From 111199c99cbb051a86bbb1081f829b6143cd449d Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 19:28:34 -0700 Subject: [PATCH 01/34] Run PR test workflow on pushes to main --- .github/workflows/pr-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 5f046e2..5193593 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -9,7 +9,7 @@ on: - ready_for_review push: branches: - - core-improvements + - main concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From ad325fc3418e4621eebca69b36bebeb50cc54a3c Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:05:08 -0700 Subject: [PATCH 02/34] Serialize workspace mutations and scope edit capture and rollback Mutating Workspace operations (apply, checkpoint creation and restore, transaction commit, archive restore, compaction and filesystem-observation refresh) now run through a per-workspace FIFO operation gate, so a failed atomic apply can no longer roll back a concurrent caller's successful edit. Edits capture and restore only the subtrees they can affect, canonicalized through ancestor symlinks, instead of the whole tree. A single write no longer reads every file, and rollback no longer deletes files that other processes created during the edit. Rollback never chmods symlinks, only sets permissions that differ, handles dangling and out-of-root links, rethrows the original error, and reports both errors via WorkspaceRollbackError when recovery itself fails. Store state is loaded before the first mutation, and recording failures roll the edit back. Retracted mutations keep a sequence high-water mark so numbers are never reused. RecordingPolicy now shapes stored history only; previews, results and events always carry diffs up to DiffOptions.default.maxTextBytes. --- README.md | 3 + Sources/Workspace/CheckpointStore.swift | 55 ++- Sources/Workspace/FS/FileSystem.swift | 13 + Sources/Workspace/Snapshot.swift | 117 ++++- Sources/Workspace/Workspace+Checkpoints.swift | 41 +- Sources/Workspace/Workspace+Edits.swift | 174 ++++++-- Sources/Workspace/Workspace+Internals.swift | 78 +++- Sources/Workspace/Workspace.swift | 10 +- Sources/Workspace/WorkspaceArchive.swift | 32 +- Sources/Workspace/WorkspaceCompaction.swift | 3 + Sources/Workspace/WorkspaceTransaction.swift | 48 +- .../WorkspaceTests/CheckpointStoreTests.swift | 24 + Tests/WorkspaceTests/EditAtomicityTests.swift | 417 ++++++++++++++++++ Tests/WorkspaceTests/EditPathTests.swift | 144 ++++++ Tests/WorkspaceTests/SnapshotTests.swift | 10 + 15 files changed, 1061 insertions(+), 108 deletions(-) create mode 100644 Tests/WorkspaceTests/EditAtomicityTests.swift create mode 100644 Tests/WorkspaceTests/EditPathTests.swift diff --git a/README.md b/README.md index 51cc31f..f6bdafd 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ print(result.changes.statistics.additions) ``` `apply` is atomic by default. `.stopOnError` and `.continueAfterError` expose partial failures explicitly. +Atomic recovery does not track hard-link aliases outside captured roots or roll back permission changes to symlink entries. Direct conveniences—`writeText`, `writeData`, `appendText`, `createDirectory`, `remove`, `copy`, `move`, link creation, and permission changes—use the same edit pipeline. @@ -169,6 +170,8 @@ Compaction rebases retained checkpoints to their nearest retained ancestor and r Directory-backed stores serialize checkpoint commits, revision reads, and compaction with a persistent lifecycle lock. Mutation history is independent and is not pruned by checkpoint retention. +Store format version 1 is retained so existing stores remain readable. Once a store contains `mutations.sequence.json`, do not use older library versions that ignore this file to write to that store, including through concurrent instances or after a downgrade: those writers can reuse retracted mutation sequence numbers. All writers must preserve the sequence high-water mark. + Directory-backed checkpoint capture also reuses existing CAS blobs when a file's size and available modification timestamp are unchanged. It still walks directory metadata for correctness, but only reads and hashes file contents that appear changed. Filesystems that do not provide modification diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 3066884..349e005 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -12,6 +12,7 @@ import Glibc /// any placeholder sequence; ``appendMutation(_:)`` returns the record with the next persisted /// monotonic number for that workspace, serialized under the mutations lock. protocol CheckpointStore: AnyObject, Sendable { + func prepare(workspaceId: UUID) async throws func changes(workspaceId: UUID) async throws -> AsyncStream func captureRevision(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint func saveRevision(_ snapshot: Snapshot, draft: CheckpointDraft) async throws -> Checkpoint @@ -29,6 +30,9 @@ protocol CheckpointStore: AnyObject, Sendable { @discardableResult func appendMutation(_ mutation: Mutation) async throws -> Mutation func listMutations(workspaceId: UUID) async throws -> [Mutation] + /// Retracts one failed operation's record without lowering the workspace's + /// sequence high-water mark. Later appends must never reuse its sequence. + func removeMutation(sequence: Int, workspaceId: UUID) async throws /// Removes mutation records with `sequence <= throughSequence`. The record carrying the /// highest sequence is always retained so later appends stay monotonic. func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws @@ -45,9 +49,12 @@ actor InMemoryCheckpointStore: CheckpointStore { private var checkpointsByWorkspace: [UUID: [Checkpoint]] = [:] private var snapshotsByWorkspace: [UUID: [UUID: Snapshot]] = [:] private var mutationsByWorkspace: [UUID: [Mutation]] = [:] + private var mutationHighWaterMarks: [UUID: Int] = [:] init() {} + func prepare(workspaceId: UUID) async throws {} + func changes(workspaceId: UUID) async throws -> AsyncStream { _ = workspaceId return AsyncStream { _ in } @@ -118,14 +125,19 @@ actor InMemoryCheckpointStore: CheckpointStore { @discardableResult func appendMutation(_ mutation: Mutation) async throws -> Mutation { var list = mutationsByWorkspace[mutation.workspaceID, default: []] - let next = (list.map(\.sequence).max() ?? 0) + 1 + let next = (mutationHighWaterMarks[mutation.workspaceID] ?? 0) + 1 var record = mutation record.sequence = next + mutationHighWaterMarks[mutation.workspaceID] = next list.append(record) mutationsByWorkspace[mutation.workspaceID] = list return record } + func removeMutation(sequence: Int, workspaceId: UUID) async throws { + mutationsByWorkspace[workspaceId]?.removeAll { $0.sequence == sequence } + } + func listMutations(workspaceId: UUID) async throws -> [Mutation] { (mutationsByWorkspace[workspaceId] ?? []).sorted { $0.sequence < $1.sequence } } @@ -252,6 +264,10 @@ actor FileCheckpointStore: CheckpointStore { self.compactEncoder = JSONEncoder() } + func prepare(workspaceId: UUID) async throws { + try ensureWorkspaceDirectories(for: workspaceId) + } + func changes(workspaceId: UUID) async throws -> AsyncStream { try ensureWorkspaceDirectories(for: workspaceId) return try DirectoryChangeStream.observe(checkpointsDirectoryURL(workspaceId: workspaceId)) @@ -730,7 +746,14 @@ actor FileCheckpointStore: CheckpointStore { lastSequence = 0 } var record = mutation - record.sequence = lastSequence + 1 + let highWaterMark = try max(lastSequence, mutationHighWaterMark(workspaceId: mutation.workspaceID)) + guard highWaterMark < Int.max else { + throw WorkspaceError.storageCorrupted("mutation sequence exhausted") + } + record.sequence = highWaterMark + 1 + // Reserve before publishing the record. A failed append may leave a + // gap, but neither a failure nor a retraction can reuse an observed ID. + try write(record.sequence, to: mutationSequenceURL(workspaceId: mutation.workspaceID)) if fileManager.fileExists(atPath: jsonl.path) { try appendJSONLLine(encode: record, to: jsonl) @@ -741,6 +764,19 @@ actor FileCheckpointStore: CheckpointStore { } } + func removeMutation(sequence: Int, workspaceId: UUID) async throws { + try Self.withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + let url = mutationsJsonlURL(workspaceId: workspaceId) + let records = try loadMutationsFromJSONL(at: url) + let highWaterMark = try max( + records.map(\.sequence).max() ?? 0, + mutationHighWaterMark(workspaceId: workspaceId) + ) + try write(highWaterMark, to: mutationSequenceURL(workspaceId: workspaceId)) + try writeAllMutationsAsJSONL(records.filter { $0.sequence != sequence }, to: url) + } + } + func listMutations(workspaceId: UUID) async throws -> [Mutation] { try validateWorkspaceFormatIfPresent(workspaceId) let jsonl = mutationsJsonlURL(workspaceId: workspaceId) @@ -1205,6 +1241,19 @@ actor FileCheckpointStore: CheckpointStore { blobsDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash, isDirectory: false) } + private func mutationSequenceURL(workspaceId: UUID) -> URL { + workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("mutations.sequence.json") + } + + /// Must be read and updated under the mutations lock. Absent for older stores. + private func mutationHighWaterMark(workspaceId: UUID) throws -> Int { + let url = mutationSequenceURL(workspaceId: workspaceId) + guard fileManager.fileExists(atPath: url.path) else { return 0 } + let value = try read(Int.self, from: url) + guard value >= 0 else { throw WorkspaceError.storageCorrupted("negative mutation sequence") } + return value + } + private func mutationsJsonlURL(workspaceId: UUID) -> URL { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("mutations.jsonl", isDirectory: false) } @@ -1240,6 +1289,8 @@ actor FileCheckpointStore: CheckpointStore { } private struct StoreFormat: Codable { + // Retained to read existing stores. Writers must preserve the sequence + // high-water mark; older writers are incompatible (see README). static let currentVersion = 1 var version: Int } diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index c0bf20a..3df5034 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -59,6 +59,19 @@ public enum WorkspaceError: Error, CustomStringConvertible, Sendable { } } +/// An operation failed and recovery also failed, either while restoring the +/// previous filesystem state or while retracting its mutation-history record. +public struct WorkspaceRollbackError: Error, CustomStringConvertible, Sendable { + /// The operation or persistence error that triggered recovery. + public let originalError: any Error + /// The error encountered while restoring files or retracting history. + public let rollbackError: any Error + + public var description: String { + "\(originalError); restoring the previous state also failed: \(rollbackError)" + } +} + /// Metadata describing a filesystem entry. public struct FileInfo: Sendable, Codable { /// The normalized path for the entry. diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index a826c9c..d5682f0 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -125,12 +125,38 @@ extension Snapshot { try await restoreEntry(snapshot.entry, to: target) } + static func infoIfPresent(at path: WorkspacePath, on target: any FileSystem) async throws -> FileInfo? { + do { + return try await target.stat(path: path) + } catch { + guard isMissingEntry(error) else { throw error } + return nil + } + } + + static func isMissingEntry(_ error: any Error) -> Bool { + if let posix = error as? POSIXError { + return posix.code == .ENOENT || posix.code == .ENOTDIR + } + let error = error as NSError + if error.domain == NSPOSIXErrorDomain { + return error.code == POSIXError.Code.ENOENT.rawValue || error.code == POSIXError.Code.ENOTDIR.rawValue + } + if error.domain == NSCocoaErrorDomain, + error.code == CocoaError.fileNoSuchFile.rawValue || error.code == CocoaError.fileReadNoSuchFile.rawValue { + return true + } + // swift-corelibs-foundation stores the underlying error as a Swift `Error`, not an `NSError`. + if let underlying = error.userInfo[NSUnderlyingErrorKey] as? any Error { + return isMissingEntry(underlying) + } + return false + } + private static func captureEntry(from target: any FileSystem, at path: WorkspacePath) async throws -> Entry { - guard await target.exists(path: path) else { + guard let info = try await infoIfPresent(at: path, on: target) else { return .missing(.init(path: path)) } - - let info = try await target.stat(path: path) switch info.kind { case .directory: let entries = try await target.listDirectory(path: path) @@ -167,8 +193,7 @@ extension Snapshot { try await ensureParentDirectory(for: file.path, on: target) let rewriteRequired: Bool - if await target.exists(path: file.path) { - let info = try await target.stat(path: file.path) + if let info = try await infoIfPresent(at: file.path, on: target) { if info.kind == .file { rewriteRequired = try await target.readFile(path: file.path) != file.data } else { @@ -182,13 +207,12 @@ extension Snapshot { if rewriteRequired { try await target.writeFile(path: file.path, data: file.data, append: false) } - try await target.setPermissions(path: file.path, permissions: file.permissions) + try await setPermissionsIfNeeded(file.permissions, at: file.path, on: target) case let .symlink(symlink): try await ensureParentDirectory(for: symlink.path, on: target) var rewriteRequired = true - if await target.exists(path: symlink.path) { - let info = try await target.stat(path: symlink.path) + if let info = try await infoIfPresent(at: symlink.path, on: target) { if info.kind == .symlink, try await target.readSymlink(path: symlink.path) == symlink.target { rewriteRequired = false } else { @@ -199,11 +223,19 @@ extension Snapshot { if rewriteRequired { try await target.createSymlink(path: symlink.path, target: symlink.target) } - try await target.setPermissions(path: symlink.path, permissions: symlink.permissions) case let .directory(directory): try await ensureDirectory(at: directory.path, permissions: directory.permissions, on: target) try await syncChildren(directory.children, under: directory.path, on: target) - try await target.setPermissions(path: directory.path, permissions: directory.permissions) + try await setPermissionsIfNeeded(directory.permissions, at: directory.path, on: target) + } + } + + private static func setPermissionsIfNeeded( + _ permissions: POSIXPermissions, at path: WorkspacePath, on target: any FileSystem + ) async throws { + let info = try await target.stat(path: path) + if info.kind != .symlink, info.permissions != permissions { + try await target.setPermissions(path: path, permissions: permissions) } } @@ -212,7 +244,11 @@ extension Snapshot { guard !parent.isRoot else { return } - try await ensureDirectory(at: parent, permissions: .defaultDirectory, on: target) + // A scoped restore owns the entry, not its existing parent. In + // particular, do not replace a symlink used as a parent directory. + if try await infoIfPresent(at: parent, on: target) == nil { + try await target.createDirectory(path: parent, recursive: true) + } } private static func ensureDirectory( @@ -220,8 +256,7 @@ extension Snapshot { permissions: POSIXPermissions, on target: any FileSystem ) async throws { - if await target.exists(path: path) { - let info = try await target.stat(path: path) + if let info = try await infoIfPresent(at: path, on: target) { if info.kind == .directory { return } @@ -231,7 +266,6 @@ extension Snapshot { if !path.isRoot { try await target.createDirectory(path: path, recursive: true) } - try await target.setPermissions(path: path, permissions: permissions) } private static func syncChildren( @@ -240,7 +274,7 @@ extension Snapshot { on target: any FileSystem ) async throws { let expectedNames = Set(expectedChildren.map { $0.path.basename }) - let existingEntries = (try? await target.listDirectory(path: parentPath)) ?? [] + let existingEntries = try await target.listDirectory(path: parentPath) for entry in existingEntries where !expectedNames.contains(entry.name) { try await target.remove(path: parentPath.appending(entry.name), recursive: true) @@ -252,9 +286,7 @@ extension Snapshot { } private static func removeEntryIfPresent(at path: WorkspacePath, on target: any FileSystem) async throws { - guard await target.exists(path: path) else { - return - } + guard let info = try await infoIfPresent(at: path, on: target) else { return } if path.isRoot { let existingEntries = try await target.listDirectory(path: path) @@ -264,7 +296,6 @@ extension Snapshot { return } - let info = try await target.stat(path: path) if info.kind == .directory { let existingEntries = try await target.listDirectory(path: path) for entry in existingEntries { @@ -287,3 +318,51 @@ private extension Array { return values } } + +// A journal holds disjoint subtrees, preserving the state before the first edit +// affecting each path. Expanding a root folds previously captured children back +// into the newly captured ancestor, rather than recording intermediate edits. +struct SnapshotJournal { + private(set) var snapshots: [Snapshot] = [] + + mutating func capture(_ paths: [WorkspacePath], from filesystem: any FileSystem) async throws { + for path in paths.sorted() { + if snapshots.contains(where: { Snapshot.contains($0.rootPath, path) }) { continue } + var snapshot = try await Snapshot.capture(from: filesystem, at: path) + for old in snapshots where Snapshot.contains(path, old.rootPath) { + snapshot.entry = Snapshot.replacing(snapshot.entry, with: old.entry) + } + snapshots.removeAll { Snapshot.contains(path, $0.rootPath) } + snapshots.append(snapshot) + } + } + + func captureAfter(from filesystem: any FileSystem) async throws -> [Snapshot] { + var result: [Snapshot] = [] + for snapshot in snapshots { + result.append(try await Snapshot.capture(from: filesystem, at: snapshot.rootPath)) + } + return result + } +} + +extension Snapshot { + static func contains(_ root: WorkspacePath, _ path: WorkspacePath) -> Bool { + root.isRoot || root == path || path.string.hasPrefix(root.string + "/") + } + + static func replacing(_ tree: Entry, with replacement: Entry) -> Entry { + if tree.path == replacement.path { return replacement } + guard case var .directory(directory) = tree, contains(tree.path, replacement.path) else { return tree } + if let index = directory.children.firstIndex(where: { contains($0.path, replacement.path) }) { + directory.children[index] = replacing(directory.children[index], with: replacement) + } else if replacement.path.dirname == tree.path { + directory.children.append(replacement) + } + if case .missing = replacement { + directory.children.removeAll { $0.path == replacement.path } + } + directory.children.sort { $0.path < $1.path } + return .directory(directory) + } +} diff --git a/Sources/Workspace/Workspace+Checkpoints.swift b/Sources/Workspace/Workspace+Checkpoints.swift index 08f80df..1fb34ed 100644 --- a/Sources/Workspace/Workspace+Checkpoints.swift +++ b/Sources/Workspace/Workspace+Checkpoints.swift @@ -3,7 +3,10 @@ import Foundation extension Workspace { /// Creates a checkpoint for the workspace's current filesystem state. public func createCheckpoint(label: String? = nil) async throws -> Checkpoint { - try await ensureLoaded() + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + try await prepareForMutation() try await reconcileCheckpointsWithStore() let checkpoint = try await store.captureRevision( from: filesystem, @@ -27,7 +30,10 @@ extension Workspace { /// The filesystem changes performed by the rollback are also recorded as a `rollback` /// mutation, so the mutation log stays a complete account of tree changes. func rollback(to checkpointId: UUID, label: String? = nil) async throws -> Checkpoint { - try await ensureLoaded() + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + try await prepareForMutation() try await reconcileCheckpointsWithStore() let checkpoint = try checkpointOrThrow(id: checkpointId) let targetSnapshot = try await loadSnapshotOrThrow( @@ -35,22 +41,23 @@ extension Workspace { workspaceId: checkpoint.workspaceId ) let previousSnapshot = try await Snapshot.capture(from: filesystem) - try await Snapshot.restore(targetSnapshot, to: filesystem) - let restoredSnapshot = try await Snapshot.capture(from: filesystem) - noteWorkspaceSnapshot(restoredSnapshot) - let changes = changeSet(from: previousSnapshot, to: restoredSnapshot) - - if !changes.isEmpty { - try await appendMutation(operation: .rollback, changes: changes) - emitWorkspaceEvent(.changes(changes)) + do { + try await Snapshot.restore(targetSnapshot, to: filesystem) + let restoredSnapshot = try await Snapshot.capture(from: filesystem) + let changes = changeSet(from: previousSnapshot, to: restoredSnapshot) + let result = try await persistMutationCheckpoint( + snapshot: restoredSnapshot, + label: label, + origin: .rollback(from: checkpoint.id), + operation: .rollback, + changes: recordingChanges(changes, from: previousSnapshot, to: restoredSnapshot) + ) + noteWorkspaceSnapshot(restoredSnapshot) + if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } + return result + } catch { + try await restoreAfterFailure([previousSnapshot], originalError: error) } - - return try await persistCheckpoint( - snapshot: restoredSnapshot, - label: label, - parentCheckpointId: headCheckpointId, - origin: .rollback(from: checkpoint.id) - ) } /// Returns one checkpoint owned by this workspace when present. diff --git a/Sources/Workspace/Workspace+Edits.swift b/Sources/Workspace/Workspace+Edits.swift index 61ddad3..7cdcbc6 100644 --- a/Sources/Workspace/Workspace+Edits.swift +++ b/Sources/Workspace/Workspace+Edits.swift @@ -23,40 +23,123 @@ extension Workspace { } /// Applies edits through one canonical execution and recording path. + /// + /// Atomic recovery restores only affected subtrees, including implicitly + /// created parents. Hard-link aliases outside those subtrees are not tracked, + /// and permission changes to symlink entries are not rolled back. + /// Direct filesystem mutations and other processes are not serialized by the + /// workspace operation gate; concurrent changes within affected roots may be lost. @discardableResult public func apply(_ edits: [Edit], policy: EditPolicy = .atomic) async throws -> EditResult { - try await ensureLoaded() - let before = try await Snapshot.capture(from: filesystem) + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + try await prepareForMutation() + var journal = SnapshotJournal() var failures: [EditFailure] = [] - for (index, edit) in edits.enumerated() { - do { - try await Self.apply(edit, to: filesystem) - } catch { - let failure = EditFailure(index: index, edit: edit, message: String(describing: error)) - switch policy { - case .atomic: - try await Snapshot.restore(before, to: filesystem) - throw error - case .stopOnError: - failures.append(failure) - break - case .continueAfterError: - failures.append(failure) - continue + do { + for (index, edit) in edits.enumerated() { + do { + let paths = try await affectedRoots(for: edit) + try await journal.capture(paths, from: filesystem) + try await Self.apply(edit, to: filesystem) + } catch { + if policy == .atomic { throw error } + failures.append(EditFailure(index: index, edit: edit, message: String(describing: error))) + if policy == .stopOnError { break } + } + } + + let after = try await journal.captureAfter(from: filesystem) + let changes = changeSet(from: journal.snapshots, to: after) + if !changes.isEmpty { + let recorded = recordingChanges(changes, from: journal.snapshots, to: after) + try await appendMutation(operation: .edit, changes: recorded) + } + for snapshot in after { noteWorkspaceSnapshot(snapshot) } + if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } + return EditResult(changes: changes, failures: failures) + } catch { + // Capture and persistence failures are also inside the atomic boundary. + try await restoreAfterFailure(journal.snapshots, originalError: error) + } + } + + /// Includes implicitly created parent directories. Hard-link aliases outside + /// these roots cannot be discovered through FileSystem and are not journaled. + private func affectedRoots(for edit: Edit) async throws -> [WorkspacePath] { + let paths: [WorkspacePath] + switch edit { + case let .replace(files, _, _): + paths = try await Self.selectedFiles(files, on: filesystem) + case let .move(source, destination): paths = [source, destination] + case let .copy(_, destination, _): paths = [destination] + case let .writeText(path, _), let .appendText(path, _), + let .writeData(path, _), let .appendData(path, _), + let .createDirectory(path, _), let .remove(path, _), + let .createSymbolicLink(path, _), let .createHardLink(path, _), + let .setPermissions(path, _): paths = [path] + } + var roots: [WorkspacePath] = [] + let resolvesPaths = await filesystem.capabilities().contains(.realPathResolution) + for path in paths { + let entryPath = try await canonicalEntryPath(path, resolvesPaths: resolvesPaths) + roots.append(try await captureRoot(for: entryPath)) + // Some backends replace the leaf link, while others follow it. Capture + // both the canonical entry and its target, but never lexical aliases. + switch edit { + case .writeText, .appendText, .writeData, .appendData, .setPermissions, .replace: + if resolvesPaths { + let target = try await resolvedCapturePath(entryPath) + roots.append(try await captureRoot(for: target)) } + default: break } } + return roots + } + + func canonicalEntryPath(_ path: WorkspacePath, resolvesPaths: Bool) async throws -> WorkspacePath { + guard resolvesPaths, !path.isRoot else { return path } + let parent = try await resolvedCapturePath(path.dirname) + return parent.appending(path.basename) + } + + // Resolve existing ancestors, preserving missing suffixes for creation edits. + // A dangling leaf still has a target even when resolveRealPath requires that + // target to exist. Only absence permits fallback; authorization errors escape. + private func resolvedCapturePath(_ path: WorkspacePath, linkDepth: Int = 0) async throws -> WorkspacePath { + guard linkDepth < 64 else { + throw WorkspaceError.invalidPath("symbolic link cycle at \(path)") + } + if path.isRoot { return try await filesystem.resolveRealPath(path: path) } + let parent = try await resolvedCapturePath(path.dirname, linkDepth: linkDepth) + let entry = parent.appending(path.basename) + guard let info = try await Snapshot.infoIfPresent(at: entry, on: filesystem) else { return entry } + guard info.kind == .symlink else { return entry } + do { + let resolved = try await filesystem.resolveRealPath(path: entry) + if resolved != entry { + return try await resolvedCapturePath(resolved, linkDepth: linkDepth + 1) + } + } catch { + guard Snapshot.isMissingEntry(error) else { throw error } + } + let target = try await filesystem.readSymlink(path: entry) + // Absolute fallback targets are interpreted as workspace paths. + let targetPath = WorkspacePath(normalizing: target, relativeTo: entry.dirname) + return try await resolvedCapturePath(targetPath, linkDepth: linkDepth + 1) + } - let after = try await Snapshot.capture(from: filesystem) - noteWorkspaceSnapshot(after) - let changes = changeSet(from: before, to: after) - if !changes.isEmpty { - try await appendMutation(operation: .edit, changes: changes) - emitWorkspaceEvent(.changes(changes)) + func captureRoot(for path: WorkspacePath) async throws -> WorkspacePath { + var root = path + while !root.dirname.isRoot, + try await Snapshot.infoIfPresent(at: root.dirname, on: filesystem) == nil { + root = root.dirname } - return EditResult(changes: changes, failures: failures) + return root } @discardableResult @@ -112,22 +195,37 @@ extension Workspace { } func changeSet(from before: Snapshot, to after: Snapshot) -> ChangeSet { - let limit: Int? - switch recording { - case let .full(maxTextBytes): limit = maxTextBytes - case .changesOnly, .off: limit = 0 - } - var result = ChangeSet.compare(before: before, after: after, maxTextBytes: limit) - if case .full = recording { - return result + changeSet(from: [before], to: [after]) + } + + func changeSet( + from before: [Snapshot], to after: [Snapshot], maxTextBytes: Int? = DiffOptions.default.maxTextBytes + ) -> ChangeSet { + let parts = zip(before, after).map { + ChangeSet.compare(before: $0, after: $1, maxTextBytes: maxTextBytes) } - result.changes = result.changes.map { change in - var copy = change - copy.diff = nil - return copy + return ChangeSet( + changes: parts.flatMap(\.changes), + textDiffOmissions: parts.flatMap(\.textDiffOmissions) + ) + } + + func recordingChanges(_ changes: ChangeSet, from before: Snapshot, to after: Snapshot) -> ChangeSet { + recordingChanges(changes, from: [before], to: [after]) + } + + func recordingChanges(_ changes: ChangeSet, from before: [Snapshot], to after: [Snapshot]) -> ChangeSet { + switch recording { + case let .full(limit): + if limit == DiffOptions.default.maxTextBytes { return changes } + return changeSet(from: before, to: after, maxTextBytes: limit) + case .changesOnly, .off: + return ChangeSet(changes: changes.changes.map { change in + var copy = change + copy.diff = nil + return copy + }) } - result.textDiffOmissions = [] - return result } private static func apply(_ edit: Edit, to fileSystem: any FileSystem) async throws { diff --git a/Sources/Workspace/Workspace+Internals.swift b/Sources/Workspace/Workspace+Internals.swift index 96002a9..2fab179 100644 --- a/Sources/Workspace/Workspace+Internals.swift +++ b/Sources/Workspace/Workspace+Internals.swift @@ -1,6 +1,37 @@ import Foundation extension Workspace { + func acquireOperation() async { + if operationInProgress { + await withCheckedContinuation { operationWaiters.append($0) } + } else { + operationInProgress = true + } + } + + func releaseOperation() { + if operationWaiters.isEmpty { + operationInProgress = false + } else { + operationWaiters.removeFirst().resume() + } + } + + func restoreAfterFailure(_ before: [Snapshot], originalError: any Error) async throws -> Never { + var rollbackError: (any Error)? + for snapshot in before { + do { + try await Snapshot.restore(snapshot, to: filesystem) + } catch { + if rollbackError == nil { rollbackError = error } + } + } + if let rollbackError { + throw WorkspaceRollbackError(originalError: originalError, rollbackError: rollbackError) + } + throw originalError + } + func ensureLoaded() async throws { if didLoadStoreState { return } if let loadTask { @@ -19,6 +50,13 @@ extension Workspace { } } + func prepareForMutation() async throws { + try await ensureLoaded() + guard !didPrepareStore else { return } + try await store.prepare(workspaceId: workspaceId) + didPrepareStore = true + } + func loadStoreState() async throws { checkpoints = try await store.listCheckpoints(workspaceId: workspaceId) mutations = try await store.listMutations(workspaceId: workspaceId) @@ -71,13 +109,39 @@ extension Workspace { return checkpoint } - func appendMutation(operation: Mutation.Operation, changes: ChangeSet) async throws { - guard recording != .off else { return } + @discardableResult + func appendMutation(operation: Mutation.Operation, changes: ChangeSet) async throws -> Mutation? { + guard recording != .off else { return nil } let persisted = try await store.appendMutation( Mutation(sequence: 0, workspaceID: workspaceId, operation: operation, changes: changes) ) mutations.append(persisted) mutations.sort { $0.sequence < $1.sequence } + return persisted + } + + // If checkpoint persistence fails after the history append, retract exactly + // our record. Other instances' mutations must not be removed by compensation. + func persistMutationCheckpoint( + snapshot: Snapshot, label: String?, origin: Checkpoint.Origin, + operation: Mutation.Operation, changes: ChangeSet + ) async throws -> Checkpoint { + let mutation = changes.isEmpty ? nil : try await appendMutation(operation: operation, changes: changes) + do { + return try await persistCheckpoint( + snapshot: snapshot, label: label, parentCheckpointId: headCheckpointId, origin: origin + ) + } catch { + if let mutation { + do { + try await store.removeMutation(sequence: mutation.sequence, workspaceId: workspaceId) + mutations.removeAll { $0.sequence == mutation.sequence } + } catch let cleanupError { + throw WorkspaceRollbackError(originalError: error, rollbackError: cleanupError) + } + } + throw error + } } func latestMutationSequence() -> Int { @@ -110,6 +174,8 @@ extension Workspace { } func refreshCheckpointEvents() async { + await acquireOperation() + defer { releaseOperation() } guard eventWatchers.values.contains(where: { $0.filter.includeCheckpoints }) else { return } guard (try? await ensureLoaded()) != nil else { return } guard let stored = try? await store.listCheckpoints(workspaceId: workspaceId) else { return } @@ -128,6 +194,8 @@ extension Workspace { } func ensureFilesystemObservation() async { + await acquireOperation() + defer { releaseOperation() } guard filesystemObservationTask == nil, let source = filesystem as? any FileSystemChangeSource else { return } @@ -145,10 +213,14 @@ extension Workspace { func noteWorkspaceSnapshot(_ snapshot: Snapshot) { guard filesystemObservationTask != nil else { return } - observedFilesystemSnapshot = snapshot + guard var observed = observedFilesystemSnapshot else { return } + observed.entry = Snapshot.replacing(observed.entry, with: snapshot.entry) + observedFilesystemSnapshot = observed } func refreshFilesystemEvents() async { + await acquireOperation() + defer { releaseOperation() } guard let before = observedFilesystemSnapshot, let after = try? await Snapshot.capture(from: filesystem) else { return } diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 61bb3ca..872d749 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -8,7 +8,9 @@ public actor Workspace { case directory(URL) } - /// Controls mutation-history detail without affecting explicit previews or diffs. + /// Controls only the detail stored in mutation history. Returned edit results, + /// events, explicit previews, and diffs are independent of this policy and use + /// `DiffOptions.default` unless the API accepts explicit diff options. public enum RecordingPolicy: Sendable, Equatable { case full(maxTextBytes: Int?) case changesOnly @@ -33,8 +35,14 @@ public actor Workspace { public nonisolated let recording: RecordingPolicy let store: any CheckpointStore + // Actor isolation alone does not span suspension points. Ownership is handed + // directly to the next waiter, preventing newcomers from overtaking the queue. + var operationInProgress = false + var operationWaiters: [CheckedContinuation] = [] + var loadTask: Task? var didLoadStoreState = false + var didPrepareStore = false var checkpoints: [Checkpoint] = [] var mutations: [Mutation] = [] var headCheckpointId: UUID? diff --git a/Sources/Workspace/WorkspaceArchive.swift b/Sources/Workspace/WorkspaceArchive.swift index f5bd6cc..b0e00b6 100644 --- a/Sources/Workspace/WorkspaceArchive.swift +++ b/Sources/Workspace/WorkspaceArchive.swift @@ -43,19 +43,29 @@ extension Workspace { /// Restores a portable archive and records the resulting changes. @discardableResult public func restore(_ archive: Archive, at destination: WorkspacePath? = nil) async throws -> ChangeSet { - try await ensureLoaded() - let before = try await Snapshot.capture(from: filesystem) - let destination = destination ?? archive.root + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + try await prepareForMutation() + let resolvesPaths = await filesystem.capabilities().contains(.realPathResolution) + let destination = try await canonicalEntryPath(destination ?? archive.root, resolvesPaths: resolvesPaths) let entry = Self.snapshotEntry(archive.entry, replacing: archive.root, with: destination) - try await Snapshot.restore(Snapshot(rootPath: destination, entry: entry), to: filesystem) - let after = try await Snapshot.capture(from: filesystem) - noteWorkspaceSnapshot(after) - let changes = changeSet(from: before, to: after) - if !changes.isEmpty { - try await appendMutation(operation: .archiveRestore, changes: changes) - emitWorkspaceEvent(.changes(changes)) + let root = try await captureRoot(for: destination) + let before = try await Snapshot.capture(from: filesystem, at: root) + do { + try await Snapshot.restore(Snapshot(rootPath: destination, entry: entry), to: filesystem) + let after = try await Snapshot.capture(from: filesystem, at: root) + let changes = changeSet(from: before, to: after) + if !changes.isEmpty { + let recorded = recordingChanges(changes, from: before, to: after) + try await appendMutation(operation: .archiveRestore, changes: recorded) + } + noteWorkspaceSnapshot(after) + if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } + return changes + } catch { + try await restoreAfterFailure([before], originalError: error) } - return changes } private static func archiveEntry(_ entry: Snapshot.Entry) -> Archive.Entry { diff --git a/Sources/Workspace/WorkspaceCompaction.swift b/Sources/Workspace/WorkspaceCompaction.swift index 81e1367..50cbd0c 100644 --- a/Sources/Workspace/WorkspaceCompaction.swift +++ b/Sources/Workspace/WorkspaceCompaction.swift @@ -67,6 +67,9 @@ extension Workspace { /// /// A dry run performs the same validation and returns projected statistics without changing storage. public func compact(retaining retention: Retention, dryRun: Bool = false) async throws -> CompactionReport { + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() try await ensureLoaded() let result = try await store.compact(workspaceId: workspaceId, retaining: retention, dryRun: dryRun) if !dryRun { diff --git a/Sources/Workspace/WorkspaceTransaction.swift b/Sources/Workspace/WorkspaceTransaction.swift index 2a8fc0a..9341f4d 100644 --- a/Sources/Workspace/WorkspaceTransaction.swift +++ b/Sources/Workspace/WorkspaceTransaction.swift @@ -220,7 +220,7 @@ extension Workspace { public func preview() async throws -> ChangeSet { try requireActive() let current = try await draft.revisionSnapshot(.current) - return ChangeSet.compare(before: base, after: current, maxTextBytes: 1_000_000) + return ChangeSet.compare(before: base, after: current, maxTextBytes: DiffOptions.default.maxTextBytes) } /// Attempts to commit this transaction. Conflicted transactions remain active. @@ -258,6 +258,9 @@ extension Workspace { /// Starts an isolated transaction from the workspace's complete current state. public func beginTransaction(label: String? = nil) async throws -> Transaction { + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() try await ensureLoaded() try await reconcileCheckpointsWithStore() let base = try await Snapshot.capture(from: filesystem) @@ -304,10 +307,13 @@ extension Workspace { label: String?, strategy: Transaction.CommitStrategy ) async throws -> Transaction.Commit { - try await ensureLoaded() + await acquireOperation() + defer { releaseOperation() } + try Task.checkCancellation() + try await prepareForMutation() try await reconcileCheckpointsWithStore() let current = try await Snapshot.capture(from: filesystem) - let preview = ChangeSet.compare(before: base, after: draft, maxTextBytes: 1_000_000) + let preview = ChangeSet.compare(before: base, after: draft, maxTextBytes: DiffOptions.default.maxTextBytes) var baseNodes: [WorkspacePath: MergeNode] = [:] var currentNodes: [WorkspacePath: MergeNode] = [:] @@ -375,27 +381,35 @@ extension Workspace { rootPath: .root, entry: Self.buildEntryTree(from: merged, rootPermissions: rootPermissions) ) - let applied = ChangeSet.compare(before: current, after: mergedSnapshot, maxTextBytes: 1_000_000) + let applied = ChangeSet.compare(before: current, after: mergedSnapshot, maxTextBytes: DiffOptions.default.maxTextBytes) guard !applied.isEmpty else { return Transaction.Commit(preview: preview, appliedChanges: applied, checkpoint: nil, conflicts: []) } + var before: [Snapshot] = [] + var after: [Snapshot] = [] + for path in applied.touchedPaths { + if before.contains(where: { Snapshot.contains($0.rootPath, path) }) { continue } + let oldEntry = Self.snapshotEntry(at: path, in: current.entry) ?? .missing(.init(path: path)) + let newEntry = Self.snapshotEntry(at: path, in: mergedSnapshot.entry) ?? .missing(.init(path: path)) + before.append(Snapshot(rootPath: path, entry: oldEntry)) + after.append(Snapshot(rootPath: path, entry: newEntry)) + } do { - try await Snapshot.restore(mergedSnapshot, to: filesystem) + for snapshot in after { try await Snapshot.restore(snapshot, to: filesystem) } + let checkpoint = try await persistMutationCheckpoint( + snapshot: mergedSnapshot, + label: label, + origin: .transaction(base: baseCheckpointID), + operation: .transaction, + changes: recordingChanges(applied, from: current, to: mergedSnapshot) + ) + for snapshot in after { noteWorkspaceSnapshot(snapshot) } + emitWorkspaceEvent(.changes(applied)) + return Transaction.Commit(preview: preview, appliedChanges: applied, checkpoint: checkpoint, conflicts: []) } catch { - try? await Snapshot.restore(current, to: filesystem) - throw error + try await restoreAfterFailure(before, originalError: error) } - try await appendMutation(operation: .transaction, changes: applied) - noteWorkspaceSnapshot(mergedSnapshot) - let checkpoint = try await persistCheckpoint( - snapshot: mergedSnapshot, - label: label, - parentCheckpointId: headCheckpointId, - origin: .transaction(base: baseCheckpointID) - ) - emitWorkspaceEvent(.changes(applied)) - return Transaction.Commit(preview: preview, appliedChanges: applied, checkpoint: checkpoint, conflicts: []) } private static func conflictDiffForTransaction(from base: MergeNode?, to side: MergeNode?) -> TextDiff? { diff --git a/Tests/WorkspaceTests/CheckpointStoreTests.swift b/Tests/WorkspaceTests/CheckpointStoreTests.swift index 56ed289..e344b5d 100644 --- a/Tests/WorkspaceTests/CheckpointStoreTests.swift +++ b/Tests/WorkspaceTests/CheckpointStoreTests.swift @@ -4,6 +4,30 @@ import Testing @Suite("Checkpoint store") struct CheckpointStoreTests { + @Test(arguments: [false, true]) + func `retraction never reuses an observed sequence`(diskBacked: Bool) async throws { + let root = try TestSupport.temporaryDirectory("SequenceRetraction") + defer { TestSupport.remove(root) } + let id = UUID() + var store: any CheckpointStore = diskBacked + ? FileCheckpointStore(rootDirectory: root) : InMemoryCheckpointStore() + let first = try await store.appendMutation(Self.sampleMutation(workspaceID: id, path: "/a")) + let retracted = try await store.appendMutation(Self.sampleMutation(workspaceID: id, path: "/b")) + try await store.removeMutation(sequence: retracted.sequence, workspaceId: id) + if diskBacked { store = FileCheckpointStore(rootDirectory: root) } + let next = try await store.appendMutation(Self.sampleMutation(workspaceID: id, path: "/c")) + let assigned: [Int] = [first.sequence, retracted.sequence, next.sequence] + #expect(assigned == [1, 2, 3]) + #expect(try await store.listMutations(workspaceId: id).map(\.sequence) == [1, 3]) + try await store.removeMutation(sequence: first.sequence, workspaceId: id) + try await store.removeMutation(sequence: next.sequence, workspaceId: id) + if diskBacked { store = FileCheckpointStore(rootDirectory: root) } + let afterEmpty = try await store.appendMutation(Self.sampleMutation(workspaceID: id, path: "/d")) + #expect(afterEmpty.sequence == 4) + let other = try await store.appendMutation(Self.sampleMutation(workspaceID: UUID(), path: "/a")) + #expect(other.sequence == 1) + } + @Test func `in-memory store isolates workspaces and assigns mutation sequences`() async throws { let store = InMemoryCheckpointStore() diff --git a/Tests/WorkspaceTests/EditAtomicityTests.swift b/Tests/WorkspaceTests/EditAtomicityTests.swift new file mode 100644 index 0000000..d2af73d --- /dev/null +++ b/Tests/WorkspaceTests/EditAtomicityTests.swift @@ -0,0 +1,417 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Edit atomicity") +struct EditAtomicityTests { + @Test + func `single write does not read unrelated files`() async throws { + let fs = EditTestFileSystem() + try await fs.base.writeFile(path: "/unrelated", data: Data("unrelated".utf8), append: false) + let workspace = Workspace(fileSystem: fs) + try await workspace.writeText("/changed", "new") + #expect(await fs.reads == ["/changed"]) + } + + @Test(arguments: [Workspace.RecordingPolicy.changesOnly, .off, .full(maxTextBytes: 0)]) + func `recording policy only shapes history`(policy: Workspace.RecordingPolicy) async throws { + let workspace = Workspace(recording: policy) + try await workspace.writeText("/a", "old\n") + let preview = try await workspace.preview([.writeText("/a", "new\n")]) + let result = try await workspace.writeText("/a", "new\n") + #expect(preview.changes.first?.diff != nil) + #expect(result.changes.first?.diff != nil) + #expect(try await workspace.history().allSatisfy { $0.changes.changes.allSatisfy { $0.diff == nil } }) + } + + @Test + func `rollback works without optional permissions support`() async throws { + let fs = EditTestFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("old".utf8), append: false) + let workspace = Workspace(fileSystem: fs) + await #expect(throws: EditTestError.self) { + try await workspace.apply([.writeText("/a", "new"), .writeText("/fail", "fail")]) + } + #expect(try await workspace.readText("/a") == "old") + #expect(try await workspace.history().isEmpty) + } + + @Test + func `local restore preserves symlink targets modes and dangling links`() async throws { + let root = try TestSupport.temporaryDirectory() + let outside = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root); TestSupport.remove(outside) } + let fs = try LocalFileSystem(root: root) + try await fs.writeFile(path: "/a", data: Data("old".utf8), append: false) + try await fs.setPermissions(path: "/a", permissions: POSIXPermissions(0o600)) + try await fs.createSymlink(path: "/link", target: "a") + try FileManager.default.createSymbolicLink(atPath: root.appendingPathComponent("dangling").path, withDestinationPath: "missing") + try FileManager.default.createSymbolicLink(atPath: root.appendingPathComponent("outside").path, withDestinationPath: outside.path) + let workspace = Workspace(fileSystem: fs) + let checkpoint = try await workspace.createCheckpoint() + try await workspace.writeText("/a", "changed") + _ = try await workspace.restore(to: checkpoint) + #expect(try await fs.stat(path: "/a").permissions == POSIXPermissions(0o600)) + #expect(try await fs.readSymlink(path: "/dangling") == "missing") + #expect(try await fs.readSymlink(path: "/outside") == outside.path) + #expect(try await workspace.readText("/a") == "old") + } + + @Test + func `external local file survives failed edit`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let fs = EditTestFileSystem(base: local) + await fs.onWrite { path in + if path == "/a" { + try Data("external".utf8).write(to: root.appendingPathComponent("external")) + } + } + let workspace = Workspace(fileSystem: fs) + await #expect(throws: EditTestError.self) { + try await workspace.apply([.writeText("/a", "new"), .writeText("/fail", "fail")]) + } + #expect(try await local.readFile(path: "/external") == Data("external".utf8)) + #expect(!(await workspace.exists("/a"))) + } + + @Test + func `persistence initialization fails before writing`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let storeURL = root.appendingPathComponent("file") + try Data().write(to: storeURL) + let fs = EditTestFileSystem() + let workspace = Workspace(fileSystem: fs, persistence: .directory(storeURL)) + await #expect(throws: (any Error).self) { try await workspace.writeText("/a", "new") } + #expect(await fs.writes.isEmpty) + #expect(!(await workspace.exists("/a"))) + } + + @Test + func `wrapped access denial during capture is never treated as absence`() async throws { + let fs = EditTestFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("original".utf8), append: false) + let denial = NSError( + domain: NSCocoaErrorDomain, + code: CocoaError.fileReadUnknown.rawValue, + userInfo: [NSUnderlyingErrorKey: NSError(domain: NSPOSIXErrorDomain, code: Int(POSIXError.Code.EACCES.rawValue))] + ) + await fs.failStat(denial) + let workspace = Workspace(fileSystem: fs) + await #expect(throws: denial) { try await workspace.writeText("/a", "new") } + #expect(await fs.writes.isEmpty) + #expect(try await fs.base.readFile(path: "/a") == Data("original".utf8)) + } + + @Test + func `post mutation capture failure rolls back`() async throws { + let fs = EditTestFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("old".utf8), append: false) + await fs.onWrite { path in if path == "/a" { await fs.failNextRead() } } + let workspace = Workspace(fileSystem: fs) + await #expect(throws: EditTestError.self) { try await workspace.writeText("/a", "new") } + #expect(try await workspace.readText("/a") == "old") + #expect(try await workspace.history().isEmpty) + } + + @Test + func `concurrent successful edit survives atomic rollback with consistent history and events`() async throws { + let fs = EditTestFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("original".utf8), append: false) + let workspace = Workspace(fileSystem: fs) + let events = await workspace.events() + let entered = EditSignal() + let resume = EditSignal() + await fs.onWrite { _ in + await entered.signal() + await resume.wait() + } + let failing = Task { + await #expect(throws: EditTestError.self) { + try await workspace.apply([.writeText("/a", "temporary"), .writeText("/fail", "fail")]) + } + } + await entered.wait() + let successful = Task { try await workspace.appendText("/a", "-success") } + // Give the second operation an opportunity to run during the first + // operation's deliberately suspended filesystem call. + for _ in 0..<100 { await Task.yield() } + await resume.signal() + _ = await failing.value + let changes = try await successful.value + #expect(try await workspace.readText("/a") == "original-success") + let history = try await workspace.history() + #expect(history.count == 1) + #expect(history.first?.changes == changes) + #expect(await nextEditEvent(in: events) == .changes(changes)) + } + + @Test + func `journal expands overlapping roots and resolves replace after preceding edits`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let workspace = Workspace(fileSystem: try LocalFileSystem(root: root)) + try await workspace.writeText("/dir/a", "old") + await #expect(throws: (any Error).self) { + try await workspace.apply([ + .writeText("/dir/a", "changed"), .remove("/dir"), + .writeText("/dir/new/deep", "created"), + .replace(files: .init(root: "/dir"), pattern: .literal("created"), with: "replaced"), + .writeText("/dir/new/deep/invalid", "fail") + ]) + } + #expect(try await workspace.readText("/dir/a") == "old") + #expect(!(await workspace.exists("/dir/new"))) + let result = try await workspace.apply([ + .writeText("/new/deep/a", "one"), + .replace(files: .init(root: "/new"), pattern: .literal("one"), with: "two") + ]) + #expect(try await workspace.readText("/new/deep/a") == "two") + #expect(result.changes.touchedPaths == ["/new", "/new/deep", "/new/deep/a"]) + } + + @Test(arguments: EditOperation.allCases) + func `recording failure restores parent for every mutation path`(operation: EditOperation) async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let workspace = Workspace(persistence: .directory(storage)) + try await workspace.writeText("/a", "checkpoint") + let checkpoint = try await workspace.createCheckpoint() + let archive = try await workspace.archive(at: "/a") + try await workspace.writeText("/a", "original") + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "transaction") + let before = try await workspace.history() + let log = storage.appendingPathComponent(workspace.workspaceID.uuidString).appendingPathComponent("mutations.jsonl") + try FileManager.default.removeItem(at: log) + try FileManager.default.createDirectory(at: log, withIntermediateDirectories: false) + await #expect(throws: (any Error).self) { + switch operation { + case .edit: _ = try await workspace.writeText("/a", "edit") + case .archive: _ = try await workspace.restore(archive) + case .checkpoint: _ = try await workspace.restore(to: checkpoint) + case .transaction: _ = try await transaction.commit() + } + } + #expect(try await workspace.readText("/a") == "original") + #expect(try await workspace.history() == before) + } + + @Test + func `checkpoint persistence failure retracts history and restores filesystem`() async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let workspace = Workspace(persistence: .directory(storage)) + try await workspace.writeText("/a", "original") + let checkpoint = try await workspace.createCheckpoint() + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "new") + let before = try await workspace.history() + let manifest = storage.appendingPathComponent(workspace.workspaceID.uuidString) + .appendingPathComponent("snapshots").appendingPathComponent("\(checkpoint.snapshotId.uuidString).json") + try FileManager.default.removeItem(at: manifest) + await #expect(throws: (any Error).self) { try await transaction.commit() } + #expect(try await workspace.readText("/a") == "original") + #expect(try await workspace.history() == before) + let reopened = Workspace(workspaceID: workspace.workspaceID, persistence: .directory(storage)) + #expect(try await reopened.history() == before) + try await reopened.writeText("/b", "after failure") + #expect(try await reopened.history().map(\.sequence) == [1, 3]) + } + + @Test(arguments: [EditOperation.transaction, .checkpoint, .archive]) + func `parent restoration serializes against concurrent apply`(operation: EditOperation) async throws { + let fs = EditTestFileSystem() + let workspace = Workspace(fileSystem: fs) + try await workspace.writeText("/a", "target") + let checkpoint = try await workspace.createCheckpoint() + let archive = try await workspace.archive(at: "/a") + try await workspace.writeText("/a", "original") + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "target") + let events = await workspace.events(.init(includeCheckpoints: false)) + let entered = EditSignal() + let resume = EditSignal() + await fs.onWrite { _ in + await entered.signal() + await resume.wait() + } + let restoring = Task { + switch operation { + case .transaction: _ = try await transaction.commit() + case .checkpoint: _ = try await workspace.restore(to: checkpoint) + case .archive: _ = try await workspace.restore(archive) + case .edit: Issue.record("restoration operation required") + } + } + await entered.wait() + let concurrent = Task { try await workspace.appendText("/a", "-success") } + // Give the second operation an opportunity to run during the first + // operation's deliberately suspended filesystem call. + for _ in 0..<100 { await Task.yield() } + await resume.signal() + try await restoring.value + let appended = try await concurrent.value + #expect(try await workspace.readText("/a") == "target-success") + let history = try await workspace.history() + let expectedOperation: Mutation.Operation = switch operation { + case .transaction: .transaction + case .checkpoint: .rollback + case .archive: .archiveRestore + case .edit: .edit + } + #expect(history.map(\.operation) == [.edit, .edit, expectedOperation, .edit]) + #expect(await nextEditEvent(in: events) == .changes(history[2].changes)) + #expect(await nextEditEvent(in: events) == .changes(appended)) + } + + @Test + func `read only first use leaves persistence uninitialized`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let storage = root.appendingPathComponent("not-created") + let workspace = Workspace(persistence: .directory(storage)) + #expect(try await workspace.history().isEmpty) + #expect(try await workspace.checkpoints().isEmpty) + let transaction = try await workspace.beginTransaction() + try await transaction.discard() + #expect(!FileManager.default.fileExists(atPath: storage.path)) + try await workspace.writeText("/a", "new") + #expect(FileManager.default.fileExists(atPath: storage.path)) + #expect(try await workspace.history().count == 1) + } + + @Test + func `default result diff limit is independent of history limit`() async throws { + let size = try #require(DiffOptions.default.maxTextBytes) + 1 + let content = String(repeating: "x", count: size) + let workspace = Workspace(recording: .full(maxTextBytes: size)) + let events = await workspace.events(.init(includeCheckpoints: false)) + let preview = try await workspace.preview([.writeText("/large", content)]) + let result = try await workspace.writeText("/large", content) + #expect(result == preview) + #expect(result.changes.first?.diff == nil) + #expect(result.textDiffOmissions == [.init(path: "/large", reason: .sizeLimitExceeded)]) + #expect(await nextEditEvent(in: events) == .changes(result)) + #expect(try await workspace.history().first?.changes.changes.first?.diff != nil) + let checkpoint = try await workspace.createCheckpoint() + let archive = try await workspace.archive(at: "/large") + try await workspace.writeText("/large", "") + _ = await nextEditEvent(in: events) + let restored = try await workspace.restore(archive) + #expect(restored.textDiffOmissions == result.textDiffOmissions) + #expect(restored.changes.first?.diff == nil) + #expect(await nextEditEvent(in: events) == .changes(restored)) + try await workspace.writeText("/large", "") + _ = await nextEditEvent(in: events) + _ = try await workspace.restore(to: checkpoint) + guard case let .changes(rollback)? = await nextEditEvent(in: events) else { + Issue.record("expected rollback change event") + return + } + #expect(rollback.textDiffOmissions == result.textDiffOmissions) + #expect(rollback.changes.first?.diff == nil) + } + + @Test + func `rollback failure carries both errors`() async throws { + let fs = EditTestFileSystem() + await fs.failRemovals() + let workspace = Workspace(fileSystem: fs) + do { + try await workspace.apply([.writeText("/a", "new"), .writeText("/fail", "fail")]) + Issue.record("expected rollback failure") + } catch let error as WorkspaceRollbackError { + #expect(error.originalError is EditTestError) + #expect(error.rollbackError is EditTestError) + #expect(error.description == "injected; restoring the previous state also failed: injected") + } + } +} + +enum EditOperation: CaseIterable, Sendable { + case edit, archive, checkpoint, transaction +} + +private enum EditTestError: Error { case injected } + +private actor EditTestFileSystem: FileSystem { + let base: any FileSystem + var reads: [WorkspacePath] = [] + var writes: [WorkspacePath] = [] + private var writeHook: (@Sendable (WorkspacePath) async throws -> Void)? + private var readFailure = false + private var statError: NSError? + func failStat(_ error: NSError) { statError = error } + private var removeFailure = false + func failRemovals() { removeFailure = true } + + init(base: any FileSystem = InMemoryFileSystem()) { self.base = base } + func onWrite(_ hook: @escaping @Sendable (WorkspacePath) async throws -> Void) { writeHook = hook } + func failNextRead() { readFailure = true } + func stat(path: WorkspacePath) async throws -> FileInfo { + if let statError { throw statError } + return try await base.stat(path: path) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await base.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func readFile(path: WorkspacePath) async throws -> Data { + reads.append(path) + if readFailure { readFailure = false; throw EditTestError.injected } + return try await base.readFile(path: path) + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + writes.append(path) + let hook = writeHook + writeHook = nil + try await hook?(path) + if path == "/fail" { throw EditTestError.injected } + try await base.writeFile(path: path, data: data, append: append) + } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { + if removeFailure { throw EditTestError.injected } + try await base.remove(path: path, recursive: recursive) + } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) + } +} + +private actor EditSignal { + private var signaled = false + private var waiters: [CheckedContinuation] = [] + func wait() async { + if signaled { return } + await withCheckedContinuation { waiters.append($0) } + } + func signal() { + signaled = true + for waiter in waiters { waiter.resume() } + waiters.removeAll() + } +} + +func nextEditEvent( + in stream: AsyncStream, timeout: Duration = .seconds(2) +) async -> WorkspaceEvent? { + await withTaskGroup(of: WorkspaceEvent?.self) { group in + group.addTask { + var iterator = stream.makeAsyncIterator() + return await iterator.next() + } + group.addTask { + try? await Task.sleep(for: timeout) + return nil + } + let result = await group.next() ?? nil + group.cancelAll() + return result + } +} diff --git a/Tests/WorkspaceTests/EditPathTests.swift b/Tests/WorkspaceTests/EditPathTests.swift new file mode 100644 index 0000000..9c74016 --- /dev/null +++ b/Tests/WorkspaceTests/EditPathTests.swift @@ -0,0 +1,144 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Edit path coverage") +struct EditPathTests { + enum Backend: CaseIterable, Sendable { case memory, local, mounted } + + struct Scenario: Sendable, CustomStringConvertible { + var description: String + var edits: [Edit] + var memoryRejects = false + } + + static let scenarios: [Scenario] = [ + .init(description: "nested mount write", edits: [.writeText("/mnt/data/x", "hi")]), + .init(description: "writeText", edits: [.writeText("/dir/a", "new")]), + .init(description: "appendText", edits: [.appendText("/dir/a", "new")]), + .init(description: "writeData", edits: [.writeData("/dir/a", Data([1, 2]))]), + .init(description: "appendData", edits: [.appendData("/dir/a", Data([1, 2]))]), + .init(description: "createDirectory", edits: [.createDirectory("/new/deep")]), + .init(description: "remove", edits: [.remove("/dir/sub")]), + .init(description: "copy", edits: [.copy(from: "/dir/sub", to: "/copied")]), + .init(description: "move", edits: [.move(from: "/dir/sub", to: "/moved")]), + .init(description: "createSymbolicLink", edits: [.createSymbolicLink("/newlink", target: "dir/a")]), + .init(description: "createHardLink", edits: [.createHardLink("/hard", target: "/dir/a")]), + .init(description: "setPermissions", edits: [.setPermissions("/dir/a", POSIXPermissions(0o600))]), + .init(description: "replace", edits: [.replace(files: .init(root: "/dir"), pattern: .literal("alpha"), with: "new")]), + .init(description: "alias writeText", edits: [.writeText("/dirlink/a", "new")]), + .init(description: "alias appendText", edits: [.appendText("/dirlink/a", "new")]), + .init(description: "alias writeData", edits: [.writeData("/dirlink/a", Data([1, 2]))]), + .init(description: "alias appendData", edits: [.appendData("/dirlink/a", Data([1, 2]))]), + // This backend currently rejects mkdir through a symlink. Verify that + // rejection remains atomic too; filesystem traversal belongs to its slice. + .init(description: "alias createDirectory", edits: [.createDirectory("/dirlink/new/deep")], memoryRejects: true), + .init(description: "alias remove", edits: [.remove("/dirlink/sub")]), + .init(description: "alias copy", edits: [.copy(from: "/dirlink/sub", to: "/dirlink/copied")]), + .init(description: "alias move", edits: [.move(from: "/dirlink/sub", to: "/dirlink/moved")]), + .init(description: "alias createSymbolicLink", edits: [.createSymbolicLink("/dirlink/newlink", target: "a")]), + .init(description: "alias createHardLink", edits: [.createHardLink("/dirlink/hard", target: "/dirlink/a")]), + .init(description: "alias setPermissions", edits: [.setPermissions("/dirlink/a", POSIXPermissions(0o600))]), + .init(description: "alias replace", edits: [.replace(files: .init(include: ["/dirlink/a"]), pattern: .literal("alpha"), with: "new")]), + .init(description: "alias new file", edits: [.writeText("/dirlink/new", "new")]), + .init(description: "leaf write", edits: [.writeText("/link", "new")]), + .init(description: "leaf append", edits: [.appendText("/link", "new")]), + .init(description: "relative leaf through alias", edits: [.writeText("/dirlink/link", "new")]), + .init(description: "dangling write", edits: [.writeText("/dangling", "new")]), + .init(description: "dangling append", edits: [.appendText("/dangling", "new")]), + .init(description: "dangling data", edits: [.writeData("/dangling", Data([1, 2]))]), + .init(description: "dangling append data", edits: [.appendData("/dangling", Data([1, 2]))]), + .init(description: "dangling chain", edits: [.writeText("/chain", "new")]), + .init(description: "remove leaf link", edits: [.remove("/dirlink/link")]), + .init(description: "move dangling link", edits: [.move(from: "/dangling", to: "/dirlink/moved")]), + .init(description: "expanded root with dangling sibling", edits: [.writeText("/dir/a", "new"), .setPermissions("/", POSIXPermissions(0o750))]), + .init(description: "move then edit alias", edits: [.move(from: "/dirlink/sub", to: "/dirlink/moved"), .writeText("/dirlink/moved/b", "new")]), + ] + + // Exercise the nested mount regression here. The backend's separate empty + // directory listing issue prevents applying the entire matrix to it yet. + static let cases = Backend.allCases.flatMap { backend in + scenarios.filter { backend != .mounted || $0.description == "nested mount write" } + .map { (backend, $0) } + } + + @Test(arguments: cases) + func `scoped changes and rollback match whole tree`(backend: Backend, scenario: Scenario) async throws { + for failing in [false, true] { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let fs: any FileSystem = switch backend { + case .memory: InMemoryFileSystem() + case .local: try LocalFileSystem(root: root) + case .mounted: + MountedFileSystem( + base: InMemoryFileSystem(), + mounts: [.init(mountPoint: "/mnt/data", fileSystem: InMemoryFileSystem())] + ) + } + if backend != .mounted { + try await fs.createDirectory(path: "/mnt/data", recursive: true) + } + try await fs.createDirectory(path: "/dir/sub", recursive: true) + try await fs.writeFile(path: "/dir/a", data: Data("alpha".utf8), append: false) + try await fs.writeFile(path: "/dir/sub/b", data: Data("beta".utf8), append: false) + try await fs.createSymlink(path: "/dirlink", target: "dir") + try await fs.createSymlink(path: "/dir/link", target: "a") + try await fs.createSymlink(path: "/link", target: "dir/a") + try await fs.createSymlink(path: "/dangling", target: "nowhere") + try await fs.createSymlink(path: "/chain", target: "dangling") + let workspace = Workspace(fileSystem: fs) + let before = try await Snapshot.capture(from: fs) + let edits = scenario.edits + (failing ? [.copy(from: "/absent", to: "/failure")] : []) + if failing || (backend != .local && scenario.memoryRejects) { + do { + try await workspace.apply(edits) + Issue.record("expected edit failure") + } catch { + #expect(!(error is WorkspaceRollbackError)) + } + let after = try await Snapshot.capture(from: fs) + #expect(after.entry == before.entry) + #expect(try await workspace.history().isEmpty) + } else { + let result = try await workspace.apply(edits) + let after = try await Snapshot.capture(from: fs) + let expected = ChangeSet.compare(before: before, after: after, maxTextBytes: DiffOptions.default.maxTextBytes) + #expect(result.changes == expected) + } + } + } + + @Test + func `removing a path beneath a local regular file is a no-op`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/dir", recursive: true) + try await fs.writeFile(path: "/dir/a", data: Data("original".utf8), append: false) + let workspace = Workspace(fileSystem: fs) + let before = try await Snapshot.capture(from: fs) + let result = try await workspace.remove("/dir/a/x") + #expect(result.isEmpty) + #expect(try await Snapshot.capture(from: fs).entry == before.entry) + #expect(try await workspace.history().isEmpty) + } + + @Test + func `canonical alias changes update the observation baseline`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/dir", recursive: true) + try await fs.createSymlink(path: "/alias", target: "dir") + let workspace = Workspace(fileSystem: fs) + let events = await workspace.watchChanges() + let changes = try await workspace.writeText("/alias/new", "new") + #expect(changes.touchedPaths == ["/dir/new"]) + #expect(await nextEditEvent(in: events) == .changes(changes)) + // Exercise the refresh directly as well as any native invalidation. This + // verifies baseline folding without depending on OS event delivery timing. + await workspace.refreshFilesystemEvents() + #expect(await nextEditEvent(in: events, timeout: .milliseconds(100)) == nil) + } +} diff --git a/Tests/WorkspaceTests/SnapshotTests.swift b/Tests/WorkspaceTests/SnapshotTests.swift index 6cc1d57..5deeee1 100644 --- a/Tests/WorkspaceTests/SnapshotTests.swift +++ b/Tests/WorkspaceTests/SnapshotTests.swift @@ -4,6 +4,16 @@ import Testing @Suite("Snapshot engine") struct SnapshotTests { + @Test + func `folding a replacement preserves unrelated missing entries`() { + let missing = Snapshot.Entry.missing(.init(path: "/unrelated")) + let file = Snapshot.Entry.file(.init(path: "/changed", data: Data(), permissions: .defaultFile)) + let tree = Snapshot.Entry.directory(.init(path: .root, permissions: .defaultDirectory, children: [file, missing])) + let replacement = Snapshot.Entry.missing(.init(path: "/changed")) + let folded = Snapshot.replacing(tree, with: replacement) + #expect(folded == .directory(.init(path: .root, permissions: .defaultDirectory, children: [missing]))) + } + @Test func `capture and restore roundtrips tree contents symlinks and permissions`() async throws { let source = InMemoryFileSystem() From 7e81580f8df7f637d944ba5919459a4c1c5d968c Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:07:31 -0700 Subject: [PATCH 03/34] Bound tool-adapter output, harden search inputs, fix CRLF diffs, validate archives workspace_search no longer traps on extreme context_lines: integers decoded from tool JSON are validated, capped, and combined with overflow-safe arithmetic. workspace_read pages by offset/max_bytes with nextOffset and truncated fields, and search gains an encoded-output budget with match_offset paging, so a single tool call can no longer return an unbounded result. Plain Workspace.search calls stay unbounded by default. TextDiff now splits lines on LF at the UTF-8 level, so CRLF files diff and patch line by line with correct statistics. Workspace.Archive.validate() checks the root, immediate-child structure and unique names of a (possibly decoded) archive before restore touches the workspace, so a malformed archive can no longer mutate outside its destination. --- Sources/Workspace/TextDiff.swift | 19 +- Sources/Workspace/WorkspaceArchive.swift | 24 ++ Sources/Workspace/WorkspaceRevision.swift | 37 +++ Sources/Workspace/WorkspaceSearch.swift | 235 ++++++++++++++-- Sources/Workspace/WorkspaceToolAdapter.swift | 150 +++++++++-- Tests/WorkspaceTests/TextDiffTests.swift | 14 + Tests/WorkspaceTests/UnifiedPatchTests.swift | 7 + .../WorkspaceArchiveValidationTests.swift | 37 +++ .../WorkspaceTests/WorkspaceSearchTests.swift | 40 +++ .../WorkspaceToolAdapterTests.swift | 250 ++++++++++++++++++ 10 files changed, 764 insertions(+), 49 deletions(-) create mode 100644 Tests/WorkspaceTests/WorkspaceArchiveValidationTests.swift create mode 100644 Tests/WorkspaceTests/WorkspaceSearchTests.swift diff --git a/Sources/Workspace/TextDiff.swift b/Sources/Workspace/TextDiff.swift index fcf8678..3f46234 100644 --- a/Sources/Workspace/TextDiff.swift +++ b/Sources/Workspace/TextDiff.swift @@ -216,17 +216,16 @@ private struct DiffLineToken: Hashable { private func diffLineTokens(in text: String) -> [DiffLineToken] { guard !text.isEmpty else { return [] } var tokens: [DiffLineToken] = [] - var current = "" - for character in text { - if character == "\n" { - tokens.append(DiffLineToken(text: current, hasTrailingNewline: true)) - current.removeAll(keepingCapacity: true) - } else { - current.append(character) - } + // CRLF is one Swift Character. Split UTF-8 code units so LF alone is the delimiter, + // preserving CR in the line's content and distinguishing a terminated final line. + let bytes = text.utf8 + var start = bytes.startIndex + for index in bytes.indices where bytes[index] == 0x0A { + tokens.append(DiffLineToken(text: String(decoding: bytes[start..() + for child in children { + guard child.path != parent, child.path.parent == parent else { + throw WorkspaceError.unsupported("invalid archive: \(child.path) is not an immediate child of \(parent)") + } + guard names.insert(child.path.name).inserted else { + throw WorkspaceError.unsupported("invalid archive: duplicate child \(child.path.name) in \(parent)") + } + pending.append(child) + } + } + } } /// Materializes a subtree from the current workspace or a checkpoint. @@ -43,6 +66,7 @@ extension Workspace { /// Restores a portable archive and records the resulting changes. @discardableResult public func restore(_ archive: Archive, at destination: WorkspacePath? = nil) async throws -> ChangeSet { + try archive.validate() await acquireOperation() defer { releaseOperation() } try Task.checkCancellation() diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index 00a3e39..5e24423 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -140,6 +140,43 @@ extension Workspace { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } + /// Returns a read path and its size without loading file contents. Current paths + /// use filesystem resolution; stat and readData must use that same resolved path. + /// Checkpoint symlinks retain revisionData's normalization and POSIX errors. + func revisionFileSize(at path: WorkspacePath, revision: Revision) async throws -> (resolvedPath: WorkspacePath, size: UInt64) { + let index: RevisionIndex + switch revision { + case .current: + let resolvedPath = try await filesystem.resolveRealPath(path: path) + let metadata = try await filesystem.stat(path: resolvedPath) + guard metadata.kind == .file else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) + } + return (resolvedPath, metadata.size) + case .checkpoint: + index = try await resolvedRevision(revision).index + } + var current = path + for _ in 0..<64 { + guard let entry = index.entry(at: current) else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) + } + if case .missing = entry { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) + } + let metadata = Self.info(for: entry) + if case let .symbolicLink(_, target, _) = entry { + current = WorkspacePath(normalizing: target, relativeTo: current.parent) + continue + } + guard metadata.kind == .file else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) + } + return (current, metadata.size) + } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) + } + public func info(_ path: WorkspacePath, at revision: Revision = .current) async throws -> FileInfo { switch revision { case .current: diff --git a/Sources/Workspace/WorkspaceSearch.swift b/Sources/Workspace/WorkspaceSearch.swift index c110dfa..76088c0 100644 --- a/Sources/Workspace/WorkspaceSearch.swift +++ b/Sources/Workspace/WorkspaceSearch.swift @@ -2,36 +2,72 @@ import Foundation public struct SearchRequest: Sendable, Codable, Equatable { public struct Limits: Sendable, Codable, Equatable { + /// Smallest supported encoded-result budget. + public static let minimumOutputBytes = 1_024 + /// Space reserved for the result envelope, including all integer digits. + static let envelopeReserve = 512 + public var maxFileBytes: UInt64 public var maxFiles: Int public var maxMatches: Int + /// Aggregate compact JSON result budget, including match text, context, and skipped-file metadata. + /// Nil leaves output unbounded; tool calls supply their own bounded budget. + public var maxOutputBytes: Int? public static let `default` = Limits(maxFileBytes: 1_000_000, maxFiles: 10_000, maxMatches: 1_000) - public init(maxFileBytes: UInt64, maxFiles: Int, maxMatches: Int) { + public init(maxFileBytes: UInt64, maxFiles: Int, maxMatches: Int, maxOutputBytes: Int? = nil) { self.maxFileBytes = maxFileBytes self.maxFiles = maxFiles self.maxMatches = maxMatches + self.maxOutputBytes = maxOutputBytes + } + + private enum CodingKeys: String, CodingKey { case maxFileBytes, maxFiles, maxMatches, maxOutputBytes } + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + maxFileBytes = try values.decode(UInt64.self, forKey: .maxFileBytes) + maxFiles = try values.decode(Int.self, forKey: .maxFiles) + maxMatches = try values.decode(Int.self, forKey: .maxMatches) + maxOutputBytes = try values.decodeIfPresent(Int.self, forKey: .maxOutputBytes) } } + /// Maximum context on each side is 100 lines; negative requests are rejected. + public static let maximumContextLines = 100 + public var files: FileSelection public var pattern: TextPattern public var contextLines: Int + /// Number of matching lines to skip, for deterministic paging within the selected files. + public var matchOffset: Int public var limits: Limits public init( pattern: TextPattern, files: FileSelection = FileSelection(), contextLines: Int = 0, - limits: Limits = .default + limits: Limits = .default, + matchOffset: Int = 0 ) { self.files = files self.pattern = pattern self.contextLines = contextLines self.limits = limits + self.matchOffset = matchOffset + } + + private enum CodingKeys: String, CodingKey { case files, pattern, contextLines, limits, matchOffset } + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + files = try values.decode(FileSelection.self, forKey: .files) + pattern = try values.decode(TextPattern.self, forKey: .pattern) + contextLines = try values.decode(Int.self, forKey: .contextLines) + limits = try values.decode(Limits.self, forKey: .limits) + matchOffset = try values.decodeIfPresent(Int.self, forKey: .matchOffset) ?? 0 } } + public struct SearchResult: Sendable, Codable, Equatable { public struct ContextLine: Sendable, Codable, Equatable { public var number: Int @@ -49,19 +85,44 @@ public struct SearchResult: Sendable, Codable, Equatable { public var line: String public var before: [ContextLine] public var after: [ContextLine] + /// True when this match's context or line was shortened to fit the output budget. + /// Use a paged file read for the complete source text. + public var truncated: Bool + /// UTF-8 bytes in the full matching line and requested context, before shortening. + public var totalBytes: UInt64 public init( path: WorkspacePath, lineNumber: Int, line: String, before: [ContextLine] = [], - after: [ContextLine] = [] + after: [ContextLine] = [], + truncated: Bool = false, + totalBytes: UInt64? = nil ) { self.path = path self.lineNumber = lineNumber self.line = line self.before = before self.after = after + self.truncated = truncated + self.totalBytes = totalBytes ?? (before + after).reduce(UInt64(line.utf8.count)) { + saturatedByteSum($0, UInt64($1.text.utf8.count)) + } + } + + private enum CodingKeys: String, CodingKey { case path, lineNumber, line, before, after, truncated, totalBytes } + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + self.init( + path: try values.decode(WorkspacePath.self, forKey: .path), + lineNumber: try values.decode(Int.self, forKey: .lineNumber), + line: try values.decode(String.self, forKey: .line), + before: try values.decode([ContextLine].self, forKey: .before), + after: try values.decode([ContextLine].self, forKey: .after), + truncated: try values.decodeIfPresent(Bool.self, forKey: .truncated) ?? false, + totalBytes: try values.decodeIfPresent(UInt64.self, forKey: .totalBytes) + ) } } @@ -83,23 +144,62 @@ public struct SearchResult: Sendable, Codable, Equatable { public enum Truncation: String, Sendable, Codable { case fileLimit case matchLimit + case outputLimit } public var matches: [Match] public var skippedFiles: [SkippedFile] public var searchedFileCount: Int public var truncation: Truncation? + public var truncated: Bool { truncation != nil } + /// Total matching lines across the searched files, excluding binary/oversized files + /// and files beyond `maxFiles`. Computing this scans every file on every page. + public var totalMatches: Int + /// Resume with this match offset. Nil means there are no remaining matching lines + /// in the searched files; individual shortened matches can still be `truncated`. + public var nextMatchOffset: Int? public init( matches: [Match], skippedFiles: [SkippedFile], searchedFileCount: Int, - truncation: Truncation? + truncation: Truncation?, + totalMatches: Int? = nil, + nextMatchOffset: Int? = nil ) { self.matches = matches self.skippedFiles = skippedFiles self.searchedFileCount = searchedFileCount self.truncation = truncation + self.totalMatches = totalMatches ?? matches.count + self.nextMatchOffset = nextMatchOffset + } + + private enum CodingKeys: String, CodingKey { + case matches, skippedFiles, searchedFileCount, truncation, truncated, totalMatches, nextMatchOffset + } + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + let matches = try values.decode([Match].self, forKey: .matches) + self.init( + matches: matches, + skippedFiles: try values.decode([SkippedFile].self, forKey: .skippedFiles), + searchedFileCount: try values.decode(Int.self, forKey: .searchedFileCount), + truncation: try values.decodeIfPresent(Truncation.self, forKey: .truncation), + totalMatches: try values.decodeIfPresent(Int.self, forKey: .totalMatches) ?? matches.count, + nextMatchOffset: try values.decodeIfPresent(Int.self, forKey: .nextMatchOffset) + ) + } + + public func encode(to encoder: Encoder) throws { + var values = encoder.container(keyedBy: CodingKeys.self) + try values.encode(matches, forKey: .matches) + try values.encode(skippedFiles, forKey: .skippedFiles) + try values.encode(searchedFileCount, forKey: .searchedFileCount) + try values.encodeIfPresent(truncation, forKey: .truncation) + try values.encode(truncated, forKey: .truncated) + try values.encode(totalMatches, forKey: .totalMatches) + try values.encodeIfPresent(nextMatchOffset, forKey: .nextMatchOffset) } } @@ -110,59 +210,143 @@ extension Workspace { } static func search(_ request: SearchRequest, on fileSystem: any FileSystem) async throws -> SearchResult { - guard request.contextLines >= 0, request.limits.maxFiles >= 0, request.limits.maxMatches >= 0 else { - throw WorkspaceError.unsupported("search limits and context must not be negative") + guard request.contextLines >= 0, request.limits.maxFiles >= 0, + request.limits.maxMatches >= 0, request.matchOffset >= 0 else { + throw WorkspaceError.unsupported("search limits, offset and context must not be negative") + } + if let budget = request.limits.maxOutputBytes, budget < SearchRequest.Limits.minimumOutputBytes { + throw WorkspaceError.unsupported("search output budget must be at least \(SearchRequest.Limits.minimumOutputBytes) bytes") } switch request.pattern { case let .literal(value, _), let .regularExpression(value): guard !value.isEmpty else { throw WorkspaceError.unsupported("search pattern must not be empty") } } + let context = min(request.contextLines, SearchRequest.maximumContextLines) let allFiles = try await selectedFiles(request.files, on: fileSystem) let files = Array(allFiles.prefix(request.limits.maxFiles)) var truncation: SearchResult.Truncation? = allFiles.count > files.count ? .fileLimit : nil var matches: [SearchResult.Match] = [] var skipped: [SearchResult.SkippedFile] = [] var searchedFileCount = 0 + var totalMatches = 0 + var pageClosed = false + // Reserve more than the maximum encoded envelope (including all integer digits). + // Charge each array element's actual JSON size plus its comma. This bounds escaping, + // paths, context and skipped-file metadata, rather than just raw text bytes. + var remaining = request.limits.maxOutputBytes.map { $0 - SearchRequest.Limits.envelopeReserve } + // Skipped files cannot consume the whole page before its first matching line. + var skippedRemaining = remaining.map { $0 / 2 } + let encoder: JSONEncoder? = request.limits.maxOutputBytes.map { _ in + let encoder = JSONEncoder() + // Measurement settings must match WorkspaceToolAdapter.call's result encoder. + encoder.outputFormatting = [.sortedKeys] + return encoder + } let regex: NSRegularExpression? = if case let .regularExpression(pattern) = request.pattern { try NSRegularExpression(pattern: pattern) - } else { - nil + } else { nil } + + func recordSkipped(_ path: WorkspacePath, reason: SearchResult.SkippedFile.Reason) throws { + let value = SearchResult.SkippedFile(path: path, reason: reason) + guard let encoder, let budget = remaining, let skippedBudget = skippedRemaining else { + skipped.append(value) + return + } + let cost = try encoder.encode(value).count + if cost < min(budget, skippedBudget) { + skipped.append(value) + remaining = budget - cost - 1 + skippedRemaining = skippedBudget - cost - 1 + } else { + truncation = .outputLimit + } } - fileLoop: for path in files { + for path in files { let info = try await fileSystem.stat(path: path) if info.size > request.limits.maxFileBytes { - skipped.append(.init(path: path, reason: .sizeLimitExceeded)) + try recordSkipped(path, reason: .sizeLimitExceeded) + continue + } + let readLimit = Int(min(request.limits.maxFileBytes, UInt64(Int.max - 1))) + 1 + let data = try await fileSystem.readFile(path: path, offset: 0, length: readLimit) + // Recheck actual bytes in case the file grew between stat and read. + if UInt64(data.count) > request.limits.maxFileBytes { + try recordSkipped(path, reason: .sizeLimitExceeded) continue } - let data = try await fileSystem.readFile(path: path) guard !data.contains(0), let text = String(data: data, encoding: .utf8) else { - skipped.append(.init(path: path, reason: .binary)) + try recordSkipped(path, reason: .binary) continue } searchedFileCount += 1 - let lines = text.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) + let lines = text.utf8.split(separator: 0x0A, omittingEmptySubsequences: false) + .map { String(decoding: $0, as: UTF8.self) } + var byteOffsets: [UInt64] = [0] + for line in lines { byteOffsets.append(byteOffsets.last! + UInt64(line.utf8.count)) } for (index, line) in lines.enumerated() where Self.matches(line, request.pattern, regex: regex) { - if matches.count >= request.limits.maxMatches { + // Clamp each distance before arithmetic; no untrusted addition/subtraction. + let lower = index - min(index, context) + let upper = index + 1 + min(lines.count - index - 1, context) + let matchBytes = byteOffsets[upper] - byteOffsets[lower] + let ordinal = totalMatches + totalMatches += 1 + guard ordinal >= request.matchOffset, !pageClosed else { continue } + guard matches.count < request.limits.maxMatches else { truncation = .matchLimit - break fileLoop + pageClosed = true + continue } - let lower = max(0, index - request.contextLines) - let upper = min(lines.count, index + request.contextLines + 1) - let before = (lower..= budget, matches.isEmpty { + // Guarantee progress for a single oversized match. Drop context first, + // then shorten the line at a UTF-8 boundary; file reads expose the rest. + match.truncated = true + match.before = [] + match.after = [] + cost = try encoder.encode(match).count + if cost >= budget { + match.line = "" + let overhead = try encoder.encode(match).count + if overhead < budget - 1 { + let bytes = Array(line.utf8.prefix((budget - overhead - 1) / 6)) + var end = bytes.count + while end > 0, String(bytes: bytes[.. UInt64 { + let sum = lhs.addingReportingOverflow(rhs) + return sum.overflow ? .max : sum.partialValue +} diff --git a/Sources/Workspace/WorkspaceToolAdapter.swift b/Sources/Workspace/WorkspaceToolAdapter.swift index 31b425b..50c4123 100644 --- a/Sources/Workspace/WorkspaceToolAdapter.swift +++ b/Sources/Workspace/WorkspaceToolAdapter.swift @@ -63,13 +63,26 @@ public actor WorkspaceToolAdapter { public init(workspace: Workspace) { self.workspace = workspace } + /// Read pages default to 64 KiB of source bytes, capped at 256 KiB. JSON escaping + /// can expand content by at most six times, plus the path and fixed metadata. + public nonisolated static let defaultReadMaxBytes = 65_536 + public nonisolated static let minimumReadMaxBytes = 4 + public nonisolated static let maximumReadMaxBytes = 262_144 + /// Search budgets bound the encoded JSON result, including context and metadata. + public nonisolated static let defaultSearchMaxBytes = 65_536 + public nonisolated static let minimumSearchMaxBytes = SearchRequest.Limits.minimumOutputBytes + public nonisolated static let maximumSearchMaxBytes = 262_144 + private nonisolated static let defaultDiffMaxBytes = DiffOptions.default.maxTextBytes ?? Int.max + public nonisolated static let definitions: [WorkspaceToolDefinition] = [ .init( name: "workspace_read", - description: "Read a UTF-8 file from the current workspace or a checkpoint.", + description: "Read a UTF-8 text file from the current workspace or a checkpoint, up to max_bytes per call. If truncated is true, call again with offset set to nextOffset.", inputSchema: .schema( properties: [ "path": .stringSchema("Absolute workspace path."), + "offset": .integerSchema("Byte offset; default 0. At/past EOF returns empty content.", minimum: 0), + "max_bytes": .integerSchema("Maximum file content bytes per call; default \(defaultReadMaxBytes), minimum \(minimumReadMaxBytes). Larger values clamp to \(maximumReadMaxBytes).", minimum: minimumReadMaxBytes, maximum: maximumReadMaxBytes), "checkpoint_id": .stringSchema("Optional checkpoint UUID."), ], required: ["path"] @@ -89,7 +102,7 @@ public actor WorkspaceToolAdapter { ), .init( name: "workspace_search", - description: "Search UTF-8 files by literal text or regular expression.", + description: "Search UTF-8 text files by literal text or regular expression in path and line order. If nextMatchOffset is present, call again with match_offset set to it. A match marked truncated was shortened; use workspace_read to read the file for its full text.", inputSchema: .schema( properties: [ "pattern": .stringSchema("Text or regular expression to find."), @@ -98,7 +111,9 @@ public actor WorkspaceToolAdapter { "root": .stringSchema("Root directory to search."), "include": .arraySchema(items: .stringSchema("Include glob.")), "exclude": .arraySchema(items: .stringSchema("Exclude glob.")), - "context_lines": .integerSchema("Context lines before and after each match."), + "context_lines": .integerSchema("Context lines on each side; default 0. Larger values clamp to \(SearchRequest.maximumContextLines).", minimum: 0, maximum: SearchRequest.maximumContextLines), + "match_offset": .integerSchema("Matching lines to skip; default 0. Continue at nextMatchOffset.", minimum: 0), + "max_bytes": .integerSchema("Maximum encoded result size in bytes; default \(defaultSearchMaxBytes), minimum \(minimumSearchMaxBytes). Larger values clamp to \(maximumSearchMaxBytes).", minimum: minimumSearchMaxBytes, maximum: maximumSearchMaxBytes), ], required: ["pattern"] ) @@ -132,7 +147,7 @@ public actor WorkspaceToolAdapter { "from_checkpoint_id": .stringSchema("Source checkpoint UUID; omit for current state."), "to_checkpoint_id": .stringSchema("Destination checkpoint UUID; omit for current state."), "unified_patch": .booleanSchema("Include git-style textual patch output."), - "max_text_bytes": .integerSchema("Maximum bytes per file for text diffing."), + "max_text_bytes": .integerSchema("Maximum bytes per file for text diffing; default and cap \(defaultDiffMaxBytes).", minimum: 0, maximum: defaultDiffMaxBytes), ] ) ), @@ -152,10 +167,31 @@ public actor WorkspaceToolAdapter { switch name { case "workspace_read": let request = try decoder.decode(ReadArguments.self, from: arguments) + let maxBytes = try boundedToolInteger(request.maxBytes, default: Self.defaultReadMaxBytes, + minimum: Self.minimumReadMaxBytes, maximum: Self.maximumReadMaxBytes, name: "max_bytes") + let offset = request.offset ?? 0 let revision = try request.revision() - return try encoder.encode( - ReadOutput(path: request.path, content: try await workspace.readText(request.path, at: revision)) - ) + let (readPath, totalBytes) = try await workspace.revisionFileSize(at: request.path, revision: revision) + let length = offset < totalBytes ? Int(min(UInt64(maxBytes), totalBytes - offset)) : 0 + let read = try await workspace.readData(from: readPath, at: revision, offset: offset, length: length) + let data = Data(read.prefix(length)) + var leadingContext = Data() + if offset > 0, let first = data.first, (0x80...0xBF).contains(first) { + let count = Int(min(offset, 3)) + leadingContext = try await workspace.readData( + from: readPath, at: revision, offset: offset - UInt64(count), length: count + ) + } + let (content, end) = try toolTextPage(data, path: request.path, + hasMore: offset < totalBytes && UInt64(data.count) < totalBytes - offset, + offset: offset, leadingContext: leadingContext) + guard end > 0 || offset >= totalBytes else { + throw WorkspaceError.unsupported("file changed or returned an incomplete read at \(request.path); retry the page") + } + // The read length was bounded by totalBytes - offset, so this cannot overflow. + let nextOffset = offset + UInt64(end) + return try encoder.encode(ReadOutput(path: request.path, content: content, offset: offset, + nextOffset: nextOffset, totalBytes: totalBytes, truncated: nextOffset < totalBytes)) case "workspace_glob": let request = try decoder.decode(GlobArguments.self, from: arguments) return try encoder.encode( @@ -167,7 +203,7 @@ public actor WorkspaceToolAdapter { ) case "workspace_search": let request = try decoder.decode(SearchArguments.self, from: arguments) - return try encoder.encode(try await workspace.search(request.request)) + return try encoder.encode(try await workspace.search(request.request())) case "workspace_apply": let request = try decoder.decode(ApplyArguments.self, from: arguments) return try encoder.encode(try await workspace.apply(try request.edits.map { try $0.edit })) @@ -176,7 +212,8 @@ public actor WorkspaceToolAdapter { let changes = try await workspace.diff( from: try request.revision(request.fromCheckpointID), to: try request.revision(request.toCheckpointID), - options: .init(maxTextBytes: request.maxTextBytes ?? DiffOptions.default.maxTextBytes) + options: .init(maxTextBytes: try boundedToolInteger(request.maxTextBytes, default: Self.defaultDiffMaxBytes, + minimum: 0, maximum: Self.defaultDiffMaxBytes, name: "max_text_bytes")) ) return try encoder.encode( DiffOutput(changes: changes, unifiedPatch: request.unifiedPatch == true ? changes.unifiedPatch() : nil) @@ -198,11 +235,24 @@ public actor WorkspaceToolAdapter { private struct ReadArguments: Decodable { var path: WorkspacePath var checkpointID: String? - enum CodingKeys: String, CodingKey { case path; case checkpointID = "checkpoint_id" } + var offset: UInt64? + var maxBytes: Int? + enum CodingKeys: String, CodingKey { + case path, offset + case checkpointID = "checkpoint_id" + case maxBytes = "max_bytes" + } func revision() throws -> Revision { try toolRevision(checkpointID) } } -private struct ReadOutput: Encodable { var path: WorkspacePath; var content: String } +private struct ReadOutput: Encodable { + var path: WorkspacePath + var content: String + var offset: UInt64 + var nextOffset: UInt64 + var totalBytes: UInt64 + var truncated: Bool +} private struct GlobArguments: Decodable { var pattern: String @@ -224,11 +274,15 @@ private struct SearchArguments: Decodable { var include: [String] = ["**/*"] var exclude: [String] = [] var contextLines = 0 + var matchOffset = 0 + var maxBytes: Int? enum CodingKeys: String, CodingKey { case pattern, root, include, exclude case regularExpression = "regular_expression" case caseSensitive = "case_sensitive" case contextLines = "context_lines" + case matchOffset = "match_offset" + case maxBytes = "max_bytes" } init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) @@ -239,12 +293,24 @@ private struct SearchArguments: Decodable { include = try container.decodeIfPresent([String].self, forKey: .include) ?? ["**/*"] exclude = try container.decodeIfPresent([String].self, forKey: .exclude) ?? [] contextLines = try container.decodeIfPresent(Int.self, forKey: .contextLines) ?? 0 + matchOffset = try container.decodeIfPresent(Int.self, forKey: .matchOffset) ?? 0 + maxBytes = try container.decodeIfPresent(Int.self, forKey: .maxBytes) } - var request: SearchRequest { - SearchRequest( + func request() throws -> SearchRequest { + let context = try boundedToolInteger(contextLines, default: 0, minimum: 0, + maximum: SearchRequest.maximumContextLines, name: "context_lines") + let offset = try boundedToolInteger(matchOffset, default: 0, minimum: 0, maximum: Int.max, name: "match_offset") + let budget = try boundedToolInteger(maxBytes, default: WorkspaceToolAdapter.defaultSearchMaxBytes, + minimum: WorkspaceToolAdapter.minimumSearchMaxBytes, + maximum: WorkspaceToolAdapter.maximumSearchMaxBytes, name: "max_bytes") + var limits = SearchRequest.Limits.default + limits.maxOutputBytes = budget + return SearchRequest( pattern: regularExpression ? .regularExpression(pattern) : .literal(pattern, caseSensitive: caseSensitive), files: .init(root: root, include: include, exclude: exclude), - contextLines: contextLines + contextLines: context, + limits: limits, + matchOffset: offset ) } } @@ -320,8 +386,11 @@ private extension JSONValue { static func booleanSchema(_ description: String) -> JSONValue { .object(["type": .string("boolean"), "description": .string(description)]) } - static func integerSchema(_ description: String) -> JSONValue { - .object(["type": .string("integer"), "description": .string(description)]) + static func integerSchema(_ description: String, minimum: Int, maximum: Int? = nil) -> JSONValue { + var value: [String: JSONValue] = ["type": .string("integer"), "description": .string(description), + "minimum": .number(Double(minimum))] + if let maximum { value["maximum"] = .number(Double(maximum)) } + return .object(value) } static func arraySchema(items: JSONValue) -> JSONValue { .object(["type": .string("array"), "items": items]) @@ -330,3 +399,52 @@ private extension JSONValue { .object(["type": .string("string"), "enum": .array(values.map(JSONValue.string))]) } } + +private func boundedToolInteger(_ value: Int?, default fallback: Int, minimum: Int, maximum: Int, name: String) throws -> Int { + let value = value ?? fallback + guard value >= minimum else { + throw WorkspaceToolError.invalidArguments("\(name) must be at least \(minimum)") + } + return min(value, maximum) +} + +// Accept a valid prefix ending before one incomplete UTF-8 scalar. Invalid bytes, +// overlong sequences, surrogates and a caller offset inside a scalar remain errors. +private func toolTextPage( + _ data: Data, path: WorkspacePath, hasMore: Bool, offset: UInt64 = 0, leadingContext: Data = Data() +) throws -> (String, Int) { + if offset > 0, let first = data.first, (0x80...0xBF).contains(first) { + let leading = data.prefix { (0x80...0xBF).contains($0) } + // Verify that these bytes complete a valid character starting before the offset. + // Orphaned continuation bytes in a malformed file remain invalidEncoding errors. + if leading.count <= 3, let start = leadingContext.lastIndex(where: { !(0x80...0xBF).contains($0) }), + String(data: Data(leadingContext[start...]) + leading, encoding: .utf8) != nil { + // Report invalidEncoding instead if the rest of the page also contains invalid bytes. + _ = try toolTextPage(Data(data.dropFirst(leading.count)), path: path, hasMore: hasMore) + throw WorkspaceToolError.invalidArguments( + "offset \(offset) is inside a UTF-8 character; use a nextOffset value from a previous read" + ) + } + throw WorkspaceError.invalidEncoding(path) + } + if let content = String(data: data, encoding: .utf8) { return (content, data.count) } + if hasMore { + for suffixLength in 1...min(3, data.count) { + let end = data.count - suffixLength + guard let content = String(data: data.prefix(end), encoding: .utf8) else { continue } + let tail = Array(data.suffix(suffixLength)) + let first = tail[0] + let width = (0xC2...0xDF).contains(first) ? 2 + : (0xE0...0xEF).contains(first) ? 3 + : (0xF0...0xF4).contains(first) ? 4 : 0 + guard tail.count < width, tail.dropFirst().allSatisfy({ (0x80...0xBF).contains($0) }) else { continue } + if tail.count > 1 { + let second = tail[1] + if (first == 0xE0 && second < 0xA0) || (first == 0xED && second > 0x9F) + || (first == 0xF0 && second < 0x90) || (first == 0xF4 && second > 0x8F) { continue } + } + return (content, end) + } + } + throw WorkspaceError.invalidEncoding(path) +} diff --git a/Tests/WorkspaceTests/TextDiffTests.swift b/Tests/WorkspaceTests/TextDiffTests.swift index 0a67bd5..9dedff4 100644 --- a/Tests/WorkspaceTests/TextDiffTests.swift +++ b/Tests/WorkspaceTests/TextDiffTests.swift @@ -112,4 +112,18 @@ struct TextDiffTests { #expect(workspaceDiff == utilityDiff) } + + @Test + func `CRLF and mixed endings retain CR content and line statistics`() throws { + let diff = TextDiff.lineBased(from: "one\r\ntwo\r\nthree\nlast\r", to: "one\r\nTWO\r\nthree\nlast\r\n") + #expect(diff.statistics == .init(originalLineCount: 4, updatedLineCount: 4, additions: 2, deletions: 2)) + let hunk = try #require(diff.hunks.first) + #expect(hunk.oldLineCount == 4) + #expect(hunk.newLineCount == 4) + #expect(hunk.lines.filter { $0.kind == .context }.map(\.text) == ["one\r", "three"]) + #expect(hunk.lines.filter { $0.kind == .removed }.map(\.text) == ["two\r", "last\r"]) + #expect(hunk.lines.filter { !$0.hasTrailingNewline }.map(\.text) == ["last\r"]) + #expect(hunk.lines.filter { $0.kind == .added }.allSatisfy { $0.hasTrailingNewline }) + #expect(TextDiff.lineBased(from: "\r\n\r\n", to: "\r\n\r\n").originalLineCount == 2) + } } diff --git a/Tests/WorkspaceTests/UnifiedPatchTests.swift b/Tests/WorkspaceTests/UnifiedPatchTests.swift index d29faa0..f80ef3a 100644 --- a/Tests/WorkspaceTests/UnifiedPatchTests.swift +++ b/Tests/WorkspaceTests/UnifiedPatchTests.swift @@ -36,4 +36,11 @@ struct UnifiedPatchTests { #expect(patch.contains("--- a/deleted.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-old\n")) #expect(!patch.contains("binary.dat")) } + + @Test + func `CRLF patch preserves bytes with accurate hunk counts and EOF marker`() { + let diff = TextDiff.lineBased(from: "one\r\ntwo\r\nlast", to: "one\r\nTWO\r\nlast\r\n") + #expect(diff.unifiedPatch(oldPath: "a/note", newPath: "b/note") == + "--- a/note\n+++ b/note\n@@ -1,3 +1,3 @@\n one\r\n-two\r\n-last\n\\ No newline at end of file\n+TWO\r\n+last\r\n") + } } diff --git a/Tests/WorkspaceTests/WorkspaceArchiveValidationTests.swift b/Tests/WorkspaceTests/WorkspaceArchiveValidationTests.swift new file mode 100644 index 0000000..c4f5045 --- /dev/null +++ b/Tests/WorkspaceTests/WorkspaceArchiveValidationTests.swift @@ -0,0 +1,37 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Archive validation") +struct WorkspaceArchiveValidationTests { + @Test + func `malformed decoded archives are rejected before any mutation`() async throws { + let workspace = Workspace() + try await workspace.createDirectory("/keep") + try await workspace.createDirectory("/Imported") + try await workspace.writeText("/keep/important", "safe") + try await workspace.writeText("/Imported/original", "original") + let before = try await workspace.archive() + let history = try await workspace.history() + let file: Workspace.Archive.Entry = .file(path: "/source/good", data: Data("new".utf8), permissions: .defaultFile) + let invalid: [Workspace.Archive] = [ + .init(root: "/source", entry: .missing("/")), + .init(root: "/source", entry: .directory(path: "/source", permissions: .defaultDirectory, children: [file, .missing("/keep")])), + .init(root: "/source", entry: .directory(path: "/source", permissions: .defaultDirectory, children: [.missing("/source/nested/child")])), + .init(root: "/source", entry: .directory(path: "/source", permissions: .defaultDirectory, children: [file, file])), + .init(root: "/", entry: .directory(path: "/", permissions: .defaultDirectory, children: [.missing("/")])), + .init(root: "/source", entry: .directory(path: "/source", permissions: .defaultDirectory, children: [.directory(path: "/source/nested", permissions: .defaultDirectory, children: [.missing("/source/escape")])])), + ] + for archive in invalid { + let decoded = try JSONDecoder().decode(Workspace.Archive.self, from: JSONEncoder().encode(archive)) + do { + _ = try await workspace.restore(decoded, at: "/Imported") + Issue.record("invalid archive was accepted") + } catch let error as WorkspaceError { + #expect(error.description.contains("invalid archive")) + } + #expect(try await workspace.archive() == before) + #expect(try await workspace.history() == history) + } + } +} diff --git a/Tests/WorkspaceTests/WorkspaceSearchTests.swift b/Tests/WorkspaceTests/WorkspaceSearchTests.swift new file mode 100644 index 0000000..ea23afa --- /dev/null +++ b/Tests/WorkspaceTests/WorkspaceSearchTests.swift @@ -0,0 +1,40 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Workspace search") +struct WorkspaceSearchTests { + @Test + func `library search is unbounded unless an output budget is supplied`() async throws { + let workspace = Workspace(recording: .off) + let line = "TODO" + String(repeating: "x", count: 80_000) + try await workspace.writeText("/note", line) + + let result = try await workspace.search(SearchRequest(pattern: .literal("TODO"))) + let match = try #require(result.matches.first) + #expect(match.line.utf8.count == 80_004) + #expect(!match.truncated) + #expect(!result.truncated) + #expect(try JSONEncoder().encode(result).count > 65_536) + + var limits = SearchRequest.Limits.default + limits.maxOutputBytes = 1_024 + let bounded = try await workspace.search(SearchRequest(pattern: .literal("TODO"), limits: limits)) + #expect(bounded.truncated) + #expect(try JSONEncoder().encode(bounded).count <= 1_024) + } + + @Test + func `search truncation is computed and encoded without aggregate byte totals`() throws { + var result = SearchResult(matches: [], skippedFiles: [], searchedFileCount: 0, truncation: nil) + result.truncation = .matchLimit + #expect(result.truncated) + let truncatedJSON = try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(result)) as? [String: Any]) + #expect(truncatedJSON["truncated"] as? Bool == true) + #expect(truncatedJSON["totalBytes"] == nil) + result.truncation = nil + #expect(!result.truncated) + let completeJSON = try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(result)) as? [String: Any]) + #expect(completeJSON["truncated"] as? Bool == false) + } +} diff --git a/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift b/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift index 39091db..4b91553 100644 --- a/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift +++ b/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift @@ -64,4 +64,254 @@ struct WorkspaceToolAdapterTests { _ = try await adapter.call(name: "missing", arguments: Data("{}".utf8)) } } + + @Test(arguments: [false, true]) + func `search clamps maximum integer context without overflow`(throughAdapter: Bool) async throws { + let workspace = Workspace() + let lines = (0..<205).map { $0 == 102 ? "TODO" : "line \($0)" } + try await workspace.writeText("/note", lines.joined(separator: "\n")) + let adapter = WorkspaceToolAdapter(workspace: workspace) + let result: SearchResult + if throughAdapter { + let data = try await adapter.call( + name: "workspace_search", + arguments: Data(#"{"pattern":"TODO","context_lines":9223372036854775807}"#.utf8) + ) + result = try JSONDecoder().decode(SearchResult.self, from: data) + } else { + result = try await workspace.search(SearchRequest(pattern: .literal("TODO"), contextLines: Int.max)) + } + let match = try #require(result.matches.first) + #expect(match.lineNumber == 103) + #expect(match.before.count == 100) + #expect(match.after.count == 100) + } + + @Test + func `numeric tool arguments reject negative fractional and out of range values`() async throws { + let adapter = WorkspaceToolAdapter(workspace: Workspace()) + for (name, json) in [ + ("workspace_search", #"{"pattern":"TODO","context_lines":-1}"#), + ("workspace_search", #"{"pattern":"TODO","context_lines":1.5}"#), + ("workspace_search", #"{"pattern":"TODO","context_lines":9223372036854775808}"#), + ("workspace_search", #"{"pattern":"TODO","match_offset":-1}"#), + ("workspace_search", #"{"pattern":"TODO","max_bytes":-1}"#), + ("workspace_diff", #"{"max_text_bytes":-1}"#), + ("workspace_diff", #"{"max_text_bytes":1.5}"#), + ("workspace_read", #"{"path":"/note","offset":-1}"#), + ("workspace_read", #"{"path":"/note","offset":1.5}"#), + ("workspace_read", #"{"path":"/note","offset":18446744073709551616}"#), + ("workspace_read", #"{"path":"/note","max_bytes":-1}"#), + ] { + await #expect(throws: WorkspaceToolError.self) { + _ = try await adapter.call(name: name, arguments: Data(json.utf8)) + } + } + } + + @Test + func `read pages are bounded lossless UTF8 and report total bytes`() async throws { + let workspace = Workspace(recording: .off) + // Keep the line count small to avoid the existing quadratic checkpoint diff cost. + let content = "abc" + String(repeating: String(repeating: "abc😀", count: 999) + "\n", count: 52) + try await workspace.writeText("/note", content) + let checkpoint = try await workspace.createCheckpoint() + let adapter = WorkspaceToolAdapter(workspace: workspace) + for checkpointID in [nil, checkpoint.id.uuidString] { + var offset: UInt64 = 0 + var reconstructed = "" + repeat { + var arguments: [String: JSONValue] = ["path": .string("/note"), "offset": .number(Double(offset))] + if let checkpointID { arguments["checkpoint_id"] = .string(checkpointID) } + let data = try await adapter.call(name: "workspace_read", arguments: JSONEncoder().encode(arguments)) + let page = try JSONDecoder().decode(ReadPage.self, from: data) + #expect(page.totalBytes == UInt64(content.utf8.count)) + #expect(page.content.utf8.count <= 65_536) + #expect(data.count < 400_000) // Includes worst-case JSON escaping and metadata. + #expect(page.offset == offset) + #expect(page.nextOffset > offset) + reconstructed += page.content + offset = page.nextOffset + if !page.truncated { break } + } while true + #expect(reconstructed == content) + } + let capped = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/note","max_bytes":9223372036854775807}"#.utf8)) + let cappedPage = try JSONDecoder().decode(ReadPage.self, from: capped) + #expect(cappedPage.content.utf8.count <= 262_144) + #expect(cappedPage.truncated) + let huge = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/note","offset":18446744073709551615,"max_bytes":9223372036854775807}"#.utf8)) + let eof = try JSONDecoder().decode(ReadPage.self, from: huge) + #expect(eof.content.isEmpty) + #expect(!eof.truncated) + } + + @Test + func `search pages bound encoded output including repeated context and escaping`() async throws { + let workspace = Workspace(recording: .off) + let line = "TODO" + String(repeating: "\u{1}", count: 500) + try await workspace.writeText("/note", Array(repeating: line, count: 50).joined(separator: "\r\n")) + let adapter = WorkspaceToolAdapter(workspace: workspace) + var offset = 0 + var lineNumbers: [Int] = [] + repeat { + let json = "{\"pattern\":\"TODO\",\"context_lines\":100,\"max_bytes\":4096,\"match_offset\":\(offset)}" + let data = try await adapter.call(name: "workspace_search", arguments: Data(json.utf8)) + #expect(data.count <= 4096) + let page = try JSONDecoder().decode(SearchPage.self, from: data) + #expect(page.totalMatches == 50) + #expect(!page.matches.isEmpty) + lineNumbers += page.matches.map(\.lineNumber) + guard let next = page.nextMatchOffset else { + #expect(page.truncated == page.matches.contains { $0.truncated }) + break + } + #expect(page.truncated) + #expect(next > offset) + offset = next + } while true + #expect(lineNumbers == Array(1...50)) + } + + @Test + func `diff caps maximum integer byte limits before comparing text`() async throws { + struct Output: Decodable { var changes: ChangeSet } + let workspace = Workspace(recording: .off) + let checkpoint = try await workspace.createCheckpoint() + try await workspace.writeText("/large", String(repeating: "x", count: 1_000_001)) + let adapter = WorkspaceToolAdapter(workspace: workspace) + let arguments = "{\"from_checkpoint_id\":\"\(checkpoint.id)\",\"max_text_bytes\":9223372036854775807}" + let data = try await adapter.call(name: "workspace_diff", arguments: Data(arguments.utf8)) + let output = try JSONDecoder().decode(Output.self, from: data) + #expect(output.changes.changes.first?.diff == nil) + #expect(output.changes.textDiffOmissions == [.init(path: "/large", reason: .sizeLimitExceeded)]) + } + + @Test + func `read rejects invalid encoding and offsets within UTF8 scalars`() async throws { + let workspace = Workspace(recording: .off) + try await workspace.writeText("/unicode", "a😀b") + // An incomplete prefix of an out-of-range scalar must not be accepted as a page. + try await workspace.writeData(Data([0x61, 0xF4, 0x90, 0x80, 0x80, 0x62]), to: "/invalid") + try await workspace.writeData(Data([0x61, 0x80, 0x62]), to: "/orphan") + let adapter = WorkspaceToolAdapter(workspace: workspace) + await #expect(throws: WorkspaceToolError.invalidArguments( + "offset 2 is inside a UTF-8 character; use a nextOffset value from a previous read" + )) { + _ = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/unicode","offset":2,"max_bytes":4}"#.utf8)) + } + for (path, arguments) in [ + (WorkspacePath("/invalid"), #"{"path":"/invalid","max_bytes":4}"#), + (WorkspacePath("/orphan"), #"{"path":"/orphan","offset":1,"max_bytes":4}"#), + ] { + do { + _ = try await adapter.call(name: "workspace_read", arguments: Data(arguments.utf8)) + Issue.record("expected invalidEncoding for malformed bytes") + } catch let error as WorkspaceError { + guard case let .invalidEncoding(invalidPath) = error else { + Issue.record("expected invalidEncoding, received \(error)") + continue + } + #expect(invalidPath == path) + } + } + let first = try JSONDecoder().decode(ReadPage.self, from: await adapter.call( + name: "workspace_read", arguments: Data(#"{"path":"/unicode","max_bytes":4}"#.utf8))) + #expect(first.content == "a") + #expect(first.nextOffset == 1) + let second = try JSONDecoder().decode(ReadPage.self, from: await adapter.call( + name: "workspace_read", arguments: Data(#"{"path":"/unicode","offset":1,"max_bytes":4}"#.utf8))) + #expect(second.content == "😀") + #expect(second.nextOffset == 5) + } + + @Test + func `read uses shared revision errors and symlink target sizes`() async throws { + let workspace = Workspace(recording: .off) + try await workspace.writeText("/note", "abcdefghijklmnopqrst") + try await workspace.createDirectory("/links") + try await workspace.createSymbolicLink("/links/note", target: "../note") + let checkpoint = try await workspace.createCheckpoint() + let adapter = WorkspaceToolAdapter(workspace: workspace) + for checkpointID in [nil, checkpoint.id.uuidString] { + let suffix = checkpointID.map { ",\"checkpoint_id\":\"\($0)\"" } ?? "" + let data = try await adapter.call(name: "workspace_read", + arguments: Data("{\"path\":\"/links/note\",\"max_bytes\":4\(suffix)}".utf8)) + let page = try JSONDecoder().decode(ReadPage.self, from: data) + #expect(page.content == "abcd") + #expect(page.totalBytes == 20) + #expect(page.nextOffset == 4) + #expect(page.truncated) + do { + _ = try await adapter.call(name: "workspace_read", + arguments: Data("{\"path\":\"/missing\"\(suffix)}".utf8)) + Issue.record("expected ENOENT for missing file") + } catch { + let posix = error as NSError + #expect(posix.domain == NSPOSIXErrorDomain) + #expect(posix.code == Int(ENOENT)) + } + } + } + + @Test + func `read resolves intermediate symlinks before measuring and reading content`() async throws { + let workspace = Workspace(recording: .off) + try await workspace.createDirectory("/x") + try await workspace.createDirectory("/y/real") + try await workspace.writeText("/x/note", "SHORT") + try await workspace.writeText("/y/note", "a much longer file body here") + try await workspace.createSymbolicLink("/x/a", target: "/y/real") + try await workspace.createSymbolicLink("/y/real/link", target: "../note") + let expected = try await workspace.readText("/x/a/link") + #expect(expected == "a much longer file body here") + + let adapter = WorkspaceToolAdapter(workspace: workspace) + let data = try await adapter.call(name: "workspace_read", + arguments: Data(#"{"path":"/x/a/link"}"#.utf8)) + let page = try JSONDecoder().decode(ReadPage.self, from: data) + #expect(page.content == expected) + #expect(page.totalBytes == UInt64(expected.utf8.count)) + #expect(page.nextOffset == UInt64(expected.utf8.count)) + #expect(!page.truncated) + } + + @Test + func `search shortens oversized lines and reserves space after skipped files`() async throws { + let workspace = Workspace(recording: .off) + for index in 0..<20 { + try await workspace.writeData(Data([0]), to: WorkspacePath("/binary-\(index)")) + } + let line = "TODO" + String(repeating: "😀", count: 20_000) + try await workspace.writeText("/z-note", line) + let adapter = WorkspaceToolAdapter(workspace: workspace) + let data = try await adapter.call(name: "workspace_search", arguments: Data(#"{"pattern":"TODO","max_bytes":1024}"#.utf8)) + #expect(data.count <= 1024) + let page = try JSONDecoder().decode(SearchResult.self, from: data) + let match = try #require(page.matches.first) + #expect(match.path == "/z-note") + #expect(match.truncated) + #expect(match.totalBytes == UInt64(line.utf8.count)) + #expect(match.line.hasPrefix("TODO")) + #expect(line.hasPrefix(match.line)) + #expect(page.totalMatches == 1) + #expect(page.nextMatchOffset == nil) + #expect(page.truncated) + let beyond = try await adapter.call(name: "workspace_search", arguments: Data(#"{"pattern":"TODO","match_offset":9223372036854775807,"max_bytes":9223372036854775807}"#.utf8)) + #expect(try JSONDecoder().decode(SearchResult.self, from: beyond).matches.isEmpty) + } + + private struct ReadPage: Decodable { + var content: String + var offset: UInt64 + var nextOffset: UInt64 + var totalBytes: UInt64 + var truncated: Bool + } + private struct SearchPage: Decodable { + var matches: [SearchResult.Match] + var totalMatches: Int + var truncated: Bool + var nextMatchOffset: Int? + } } From df52c6e59594da220fb7d0755388ae8045132b00 Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:08:49 -0700 Subject: [PATCH 04/34] Bind root jail and authorization to the paths operations actually touch LocalFileSystem now resolves paths descriptor-relative from a pinned root descriptor, walking components without following symlinks implicitly and performing each operation with *at() calls on the resolved parent, so an ancestor swapped for an outside symlink between check and use can no longer escape the root. Moves use an exclusive rename where the filesystem supports it, falling back to check-then-rename on the same pinned descriptors. resolveRealPath maps through realpath and no longer collapses to "/" when the root has a symlinked spelling, and chunked reads are demand-driven. AuthorizedFileSystem authorizes the resolved target as well as the lexical path for dereferencing operations, so a symlink under an allowed prefix can no longer reach a denied file. Deny rules now match if any endpoint of a copy, move or hard link matches; allow rules still require all endpoints. OverlayFileSystem resolves every path component through the merged namespace, so whiteouts and copy-up can't be bypassed via an ancestor symlink. --- Sources/Workspace/FS/LocalFileSystem.swift | 921 +++++++++++------- Sources/Workspace/FS/OverlayFileSystem.swift | 97 +- Sources/Workspace/Support/Permissions.swift | 120 ++- Tests/WorkspaceTests/FilesystemTests.swift | 6 +- .../WorkspaceTests/PathResolutionTests.swift | 569 +++++++++++ 5 files changed, 1309 insertions(+), 404 deletions(-) create mode 100644 Tests/WorkspaceTests/PathResolutionTests.swift diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 6fd6199..5a90212 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -2,31 +2,71 @@ import Foundation #if canImport(Darwin) import Darwin +private typealias LocalStat = Darwin.stat #if os(macOS) import CoreServices #endif #elseif canImport(Glibc) import Glibc +private typealias LocalStat = Glibc.stat #endif /// A disk-backed filesystem rooted at a concrete directory on the host filesystem. /// -/// Paths are resolved relative to the configured root and constrained so callers cannot escape that root -/// through traversal or symlink resolution. +/// Operations retain the configured root descriptor and walk directories with `O_NOFOLLOW`. +/// Symlinks are expanded explicitly, with a 40-hop limit; final syscalls never follow links. +/// This is a safety layer, not a sandbox: a host process can relocate an already-open directory +/// outside the root, change hard-linked data, or mutate entries during recursive operations. +/// Watch subscriptions still use the configured URL and are not descriptor-pinned. public final class LocalFileSystem: FileSystem, @unchecked Sendable { + /// Returns zero on success or a captured POSIX errno on failure. Injection is per-instance + /// so unsupported filesystem behavior can be tested without mounts or global syscall hooks. + typealias ExclusiveRename = @Sendable (Int32, String, Int32, String) -> Int32 + private let fileManager: FileManager + private let exclusiveRename: ExclusiveRename private let stateLock = NSLock() - private var rootURL: URL? - private var resolvedRootPath: String? + private var configuration: ConfigurationSnapshot? + + private final class Descriptor: @unchecked Sendable { + let value: Int32 + init(_ value: Int32) { + self.value = value + } + deinit { + close(value) + } + } private struct ConfigurationSnapshot { - var rootURL: URL - var resolvedRootPath: String + let rootURL: URL + let resolvedRootPath: String + let foundationRootPath: String + let root: Descriptor + } + + // Glibc does not expose O_PATH without _GNU_SOURCE. + private static var searchDirectoryFlags: Int32 { + #if canImport(Darwin) + O_SEARCH | O_DIRECTORY + #else + 0x0020_0000 | O_DIRECTORY // Linux O_PATH, from asm-generic/fcntl.h. + #endif + } + + private struct Entry { + let parent: Descriptor + let name: String + let path: WorkspacePath } /// Creates an unconfigured disk-backed filesystem. - init(fileManager: FileManager = .default) { + init( + fileManager: FileManager = .default, + exclusiveRename: @escaping ExclusiveRename = LocalFileSystem.renameExclusively + ) { self.fileManager = fileManager + self.exclusiveRename = exclusiveRename } /// Creates and configures a disk-backed filesystem rooted at `rootDirectory`. @@ -35,7 +75,7 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { try applyConfiguration(rootDirectory: rootDirectory) } - /// Creates a disk-backed filesystem with an immutable public root. + /// Creates a disk-backed filesystem with a retained root directory descriptor. public convenience init(root: URL, fileManager: FileManager = .default) throws { try self.init(rootDirectory: root, fileManager: fileManager) } @@ -45,93 +85,271 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { try applyConfiguration(rootDirectory: rootDirectory) } - /// Applies configuration synchronously; also used by adapters that cannot use `async` initializers. + /// Applies configuration synchronously for adapters with synchronous initializers. internal func applyConfiguration(rootDirectory: URL) throws { + let url = rootDirectory.standardizedFileURL + try fileManager.createDirectory(at: url, withIntermediateDirectories: true) + guard let physical = realpath(url.path, nil) else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + defer { + free(physical) + } + let canonical = String(cString: physical) + let rootFD = open(canonical, Self.searchDirectoryFlags | O_NOFOLLOW | O_CLOEXEC) + let openError = errno + try check(rootFD, errorCode: openError) + let root = Descriptor(rootFD) + stateLock.lock() + defer { + stateLock.unlock() + } + configuration = ConfigurationSnapshot( + rootURL: url, + resolvedRootPath: canonical, + foundationRootPath: url.resolvingSymlinksInPath().standardizedFileURL.path, + root: root + ) + } + + private func requireConfiguration() throws -> ConfigurationSnapshot { stateLock.lock() - defer { stateLock.unlock() } - let standardized = rootDirectory.standardizedFileURL - try fileManager.createDirectory(at: standardized, withIntermediateDirectories: true) - let resolved = standardized.resolvingSymlinksInPath().standardizedFileURL - rootURL = standardized - resolvedRootPath = resolved.path + defer { + stateLock.unlock() + } + guard let configuration else { + throw WorkspaceError.unsupported("local filesystem requires a root") + } + return configuration + } + + private func check(_ result: Int32, errorCode: Int32 = errno) throws { + guard result >= 0 else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errorCode)) + } + } + + private func metadata(_ entry: Entry) throws -> LocalStat { + var value = LocalStat() + try check(fstatat(entry.parent.value, entry.name, &value, AT_SYMLINK_NOFOLLOW)) + return value + } + + private func linkTarget(_ entry: Entry) throws -> String { + var bytes = [UInt8](repeating: 0, count: 256) + while true { + let count = bytes.withUnsafeMutableBytes { + readlinkat( + entry.parent.value, entry.name, $0.baseAddress!.assumingMemoryBound(to: CChar.self), $0.count) + } + guard count >= 0 else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + if count < bytes.count { + return String(decoding: bytes.prefix(count), as: UTF8.self) + } + bytes = [UInt8](repeating: 0, count: bytes.count * 2) + } + } + + /// Absolute disk symlink targets are host paths. Accept physical, Foundation-resolved, and + /// configured root spellings, mapping each to the retained descriptor with exact boundaries. + private func absoluteComponents(_ target: String, configuration: ConfigurationSnapshot) throws -> [String] { + for root in [configuration.resolvedRootPath, configuration.foundationRootPath, configuration.rootURL.path] { + if target == root { + return [] + } + let prefix = root == "/" ? "/" : root + "/" + if target.hasPrefix(prefix) { + return String(target.dropFirst(prefix.count)).split(separator: "/").map(String.init) + } + } + throw WorkspaceError.invalidPath(target) + } + + private func resolve( + _ path: WorkspacePath, followFinal: Bool, createParents: Bool = false, + configuration: ConfigurationSnapshot? = nil + ) throws -> Entry { + let config = try configuration ?? requireConfiguration() + guard !path.string.contains("\u{0}") else { + throw WorkspaceError.invalidPath(path.string) + } + var pending = path.components + var current = config.root + var names: [String] = [] + var hops = 0 + while !pending.isEmpty { + let name = pending.removeFirst() + if name == "." || name.isEmpty { + continue + } + if name == ".." { + guard !names.isEmpty else { + throw WorkspaceError.invalidPath(path.string) + } + names.removeLast() + current = config.root + for ancestor in names { + let entry = Entry(parent: current, name: ancestor, path: .root) + current = try openEntry(entry, flags: Self.searchDirectoryFlags) + } + continue + } + let final = pending.isEmpty + let entry = Entry( + parent: current, name: name, + path: WorkspacePath(unchecked: "/" + (names + [name]).joined(separator: "/"))) + if final && !followFinal { + return entry + } + var info: LocalStat + do { + info = try metadata(entry) + } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { + if final { + return entry + } + guard createParents else { + throw error + } + let result = mkdirat(entry.parent.value, name, 0o755) + if result < 0 && errno != EEXIST { + try check(result) + } + info = try metadata(entry) + } + if info.st_mode & S_IFMT == S_IFLNK { + hops += 1 + guard hops <= 40 else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) + } + let target = try linkTarget(entry) + if target.hasPrefix("/") { + pending = try absoluteComponents(target, configuration: config) + pending + current = config.root + names = [] + } else { + pending = target.split(separator: "/").map(String.init) + pending + } + continue + } + if final { + return entry + } + current = try openEntry(entry, flags: Self.searchDirectoryFlags) + names.append(name) + } + return Entry(parent: current, name: ".", path: WorkspacePath(unchecked: "/" + names.joined(separator: "/"))) + } + + private func openEntry(_ entry: Entry, flags: Int32, mode: mode_t = 0o666) throws -> Descriptor { + let fd = openat(entry.parent.value, entry.name, flags | O_NOFOLLOW | O_CLOEXEC, mode) + let openError = errno + try check(fd, errorCode: openError) + return Descriptor(fd) + } + + private func fileInfo(_ info: LocalStat, path: WorkspacePath) -> FileInfo { + let type = info.st_mode & S_IFMT + #if canImport(Darwin) + let modified = info.st_mtimespec + #else + let modified = info.st_mtim + #endif + return FileInfo( + path: path, kind: type == S_IFLNK ? .symlink : (type == S_IFDIR ? .directory : .file), + size: UInt64(max(0, info.st_size)), permissions: POSIXPermissions(Int(info.st_mode & 0o7777)), + modificationDate: Date(timeIntervalSince1970: Double(modified.tv_sec) + Double(modified.tv_nsec) / 1e9)) } /// See ``FileSystem/stat(path:)``. /// - /// Uses `lstat` semantics: a symlink is reported as a symlink without resolving its target, - /// so links pointing outside the root can still be inspected. + /// Uses lstat semantics: inspect the link itself, including dangling or external links. public func stat(path: WorkspacePath) async throws -> FileInfo { - let configuration = try requireConfiguration() - let url = try noFollowURL(for: path, configuration: configuration) - let attributes = try fileManager.attributesOfItem(atPath: url.path) - - let fileType = attributes[.type] as? FileAttributeType - let isDirectory = fileType == .typeDirectory - let isSymbolicLink = fileType == .typeSymbolicLink - let size = (attributes[.size] as? NSNumber)?.uint64Value ?? 0 - let permissionBits = (attributes[.posixPermissions] as? NSNumber)?.intValue ?? 0 - let modificationDate = attributes[.modificationDate] as? Date - - let kind: FileTree.Kind - if isSymbolicLink { - kind = .symlink - } else if isDirectory { - kind = .directory - } else { - kind = .file - } + try fileInfo(metadata(resolve(path, followFinal: false)), path: path) + } - return FileInfo( - path: path, - kind: kind, - size: size, - permissions: POSIXPermissions(permissionBits), - modificationDate: modificationDate - ) + private func names(in directory: Descriptor) throws -> [String] { + // openat(".") gives an independent directory offset; dup would share it. + let fd = openat(directory.value, ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC) + guard fd >= 0 else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + guard let stream = fdopendir(fd) else { + let error = errno + close(fd) + throw NSError(domain: NSPOSIXErrorDomain, code: Int(error)) + } + defer { + closedir(stream) + } + var result: [String] = [] + while true { + errno = 0 + guard let item = readdir(stream) else { + if errno != 0 { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + break + } + let name = withUnsafePointer(to: &item.pointee.d_name) { + $0.withMemoryRebound(to: CChar.self, capacity: MemoryLayout.size(ofValue: item.pointee.d_name)) { + String(cString: $0) + } + } + if name != "." && name != ".." { + result.append(name) + } + } + return result.sorted() } /// See ``FileSystem/listDirectory(path:)``. public func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { - let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) - var isDirectory: ObjCBool = false - guard fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory), isDirectory.boolValue else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) + let directory = try openEntry(resolve(path, followFinal: true), flags: O_RDONLY | O_DIRECTORY) + return try names(in: directory).map { name in + let child = path.appending(name) + return DirectoryEntry( + name: name, info: fileInfo(try metadata(Entry(parent: directory, name: name, path: child)), path: child) + ) } + } - let names = try fileManager.contentsOfDirectory(atPath: url.path).sorted() - var entries: [DirectoryEntry] = [] - entries.reserveCapacity(names.count) - for name in names { - let childPath = path.appending(name) - let info = try stat(path: childPath, configuration: configuration) - entries.append(DirectoryEntry(name: name, info: info)) + private func readHandle(_ path: WorkspacePath) throws -> FileHandle { + let fd = try openEntry(resolve(path, followFinal: true), flags: O_RDONLY | O_NONBLOCK) + var info = LocalStat() + try check(fstat(fd.value, &info)) + guard info.st_mode & S_IFMT == S_IFREG else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(info.st_mode & S_IFMT == S_IFDIR ? EISDIR : EINVAL)) } - return entries + return FileHandle(fileDescriptor: try duplicateDescriptor(fd.value), closeOnDealloc: true) } - /// See ``FileSystem/readFile(path:)``. - public func readFile(path: WorkspacePath) async throws -> Data { - let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) - return try Data(contentsOf: url) + private func duplicateDescriptor(_ fd: Int32) throws -> Int32 { + let copy = fcntl(fd, F_DUPFD_CLOEXEC, 0) + try check(copy) + return copy } - /// See ``FileSystem/capabilities()``. - public func capabilities() async -> FileSystemFeatures { - [.symlinks, .hardLinks, .permissions, .realPathResolution] + /// See ``FileSystem/readFile(path:)``. + public func readFile(path: WorkspacePath) async throws -> Data { + let handle = try readHandle(path) + defer { + try? handle.close() + } + return try handle.readToEnd() ?? Data() } - /// See ``FileSystem/readFile(path:offset:length:)``. Reads only the requested range using a - /// seeking file handle instead of loading the whole file. + /// See ``FileSystem/readFile(path:offset:length:)``. public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { if let length, length < 0 { throw WorkspaceError.unsupported("read length must not be negative") } - let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) - let handle = try FileHandle(forReadingFrom: url) - defer { try? handle.close() } + let handle = try readHandle(path) + defer { + try? handle.close() + } let size = try handle.seekToEnd() guard offset < size else { return Data() @@ -143,377 +361,341 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { return try handle.readToEnd() ?? Data() } - /// See ``FileSystem/readFileChunks(path:chunkSize:)``. Streams the file incrementally so - /// large files never need to be fully resident. - public func readFileChunks( - path: WorkspacePath, - chunkSize: Int - ) async throws -> AsyncThrowingStream { - guard chunkSize > 0 else { - throw WorkspaceError.unsupported("chunk size must be positive") + private actor ChunkReader { + private var handle: FileHandle? + init(_ handle: FileHandle) { + self.handle = handle } - let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) - // Validate readability up front so the caller sees open errors directly. - let probe = try FileHandle(forReadingFrom: url) - try probe.close() - - return AsyncThrowingStream { continuation in - let reader = Task.detached { - do { - let handle = try FileHandle(forReadingFrom: url) - defer { try? handle.close() } - while !Task.isCancelled { - guard let chunk = try handle.read(upToCount: chunkSize), !chunk.isEmpty else { - break - } - continuation.yield(chunk) - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } + func next(size: Int) throws -> Data? { + guard let handle else { + return nil } - continuation.onTermination = { _ in - reader.cancel() + do { + try Task.checkCancellation() + if let data = try handle.read(upToCount: size), !data.isEmpty { + return data + } + try handle.close() + self.handle = nil + return nil + } catch { + try? handle.close() + self.handle = nil + throw error } } } - /// See ``FileSystem/createFile(path:data:)``. Uses an exclusive write so concurrent creators - /// cannot overwrite each other. - public func createFile(path: WorkspacePath, data: Data) async throws { - let configuration = try requireConfiguration() - let url = try creationURL(for: path, configuration: configuration) - let parent = url.deletingLastPathComponent() - try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) - do { - try data.write(to: url, options: [.withoutOverwriting]) - } catch { - let nsError = error as NSError - if nsError.domain == NSCocoaErrorDomain, nsError.code == CocoaError.fileWriteFileExists.rawValue { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(EEXIST)) - } - throw error + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. + /// + /// Opens once, then reads at most one chunk per consumer request. No producer or byte queue. + public func readFileChunks(path: WorkspacePath, chunkSize: Int) async throws -> AsyncThrowingStream { + guard chunkSize > 0 else { + throw WorkspaceError.unsupported("chunk size must be positive") } + let reader = ChunkReader(try readHandle(path)) + return AsyncThrowingStream(unfolding: { + try await reader.next(size: chunkSize) + }) } - /// See ``FileSystem/writeFile(path:data:append:)``. - public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { - let configuration = try requireConfiguration() - let url = try creationURL(for: path, configuration: configuration) + private func write(_ data: Data, to descriptor: Descriptor) throws { + let handle = FileHandle(fileDescriptor: try duplicateDescriptor(descriptor.value), closeOnDealloc: true) + defer { + try? handle.close() + } + try handle.write(contentsOf: data) + } - let parent = url.deletingLastPathComponent() - try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + let entry = try resolve(path, followFinal: false, createParents: true) + try write(data, to: openEntry(entry, flags: O_WRONLY | O_CREAT | O_EXCL)) + } - if append, fileManager.fileExists(atPath: url.path) { - let handle = try FileHandle(forWritingTo: url) - defer { try? handle.close() } - try handle.seekToEnd() - try handle.write(contentsOf: data) + /// See ``FileSystem/writeFile(path:data:append:)``. + /// + /// Replacement is atomic and breaks hard-link sharing; append preserves the opened inode. + /// Symlink targets are resolved explicitly before either operation. + public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + let entry = try resolve(path, followFinal: true, createParents: true) + if append { + try write(data, to: openEntry(entry, flags: O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK)) } else { - try data.write(to: url, options: .atomic) + let existing: LocalStat? + do { + existing = try metadata(entry) + } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { + existing = nil + } + try replace(entry) { output in + try self.write(data, to: output) + if let existing, existing.st_mode & S_IFMT == S_IFREG { + try check(fchmod(output.value, existing.st_mode & 0o7777)) + } + } + } + } + + private func replace(_ entry: Entry, body: (Descriptor) throws -> Void) throws { + let temporary = ".workspace-" + UUID().uuidString + let temporaryEntry = Entry(parent: entry.parent, name: temporary, path: entry.path) + let fd = try openEntry(temporaryEntry, flags: O_WRONLY | O_CREAT | O_EXCL) + defer { + _ = unlinkat(entry.parent.value, temporary, 0) } + try body(fd) + try check(renameat(entry.parent.value, temporary, entry.parent.value, entry.name)) } /// See ``FileSystem/createDirectory(path:recursive:)``. public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { - let configuration = try requireConfiguration() - let url = try creationURL(for: path, configuration: configuration) - try fileManager.createDirectory(at: url, withIntermediateDirectories: recursive) + let entry = try resolve(path, followFinal: true, createParents: recursive) + let result = mkdirat(entry.parent.value, entry.name, 0o755) + if result < 0 && errno == EEXIST && recursive { + _ = try openEntry(entry, flags: Self.searchDirectoryFlags) + return + } + try check(result) } - /// See ``FileSystem/remove(path:recursive:)``. - /// - /// Uses `lstat` semantics: removing a symlink unlinks the link itself — never its target — - /// so dangling links and links pointing outside the root can be cleaned up. - public func remove(path: WorkspacePath, recursive: Bool) async throws { - let configuration = try requireConfiguration() - let url = try noFollowURL(for: path, configuration: configuration) - - guard let attributes = try? fileManager.attributesOfItem(atPath: url.path) else { + private func removeEntry(_ entry: Entry, recursive: Bool) throws { + let info: LocalStat + do { + info = try metadata(entry) + } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { return } - - if attributes[.type] as? FileAttributeType == .typeDirectory, !recursive { - let contents = try fileManager.contentsOfDirectory(atPath: url.path) - if !contents.isEmpty { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTEMPTY)) + if info.st_mode & S_IFMT == S_IFDIR { + if recursive { + let directory = try openEntry(entry, flags: O_RDONLY | O_DIRECTORY) + for name in try names(in: directory) { + try removeEntry( + Entry(parent: directory, name: name, path: entry.path.appending(name)), recursive: true) + } } + try check(unlinkat(entry.parent.value, entry.name, AT_REMOVEDIR)) + } else { + try check(unlinkat(entry.parent.value, entry.name, 0)) } + } - try fileManager.removeItem(at: url) + /// See ``FileSystem/remove(path:recursive:)``. + /// + /// Unlinks the entry itself; never follows a final symlink, even for recursive removal. + public func remove(path: WorkspacePath, recursive: Bool) async throws { + guard !path.isRoot else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EPERM)) + } + do { + try removeEntry(resolve(path, followFinal: false), recursive: recursive) + } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { + return + } } /// See ``FileSystem/move(from:to:)``. + /// + /// Uses exclusive rename where supported. Otherwise, destination checking followed by plain + /// rename provides best-effort no-clobber behavior: a concurrent creator can still be replaced. + /// Both paths remain descriptor-pinned in either case. public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { - let configuration = try requireConfiguration() - let source = try noFollowURL(for: sourcePath, configuration: configuration) - let destination = try creationURL(for: destinationPath, configuration: configuration) - let parent = destination.deletingLastPathComponent() - try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) - try fileManager.moveItem(at: source, to: destination) + let config = try requireConfiguration() + let source = try resolve(sourcePath, followFinal: false, configuration: config) + let destination = try resolve(destinationPath, followFinal: false, createParents: true, configuration: config) + let renameError = exclusiveRename(source.parent.value, source.name, destination.parent.value, destination.name) + if renameError == 0 { + return + } + #if canImport(Darwin) + let requiresFallback = renameError == ENOTSUP || renameError == ENOSYS + #else + let requiresFallback = renameError == EINVAL || renameError == ENOSYS + #endif + guard requiresFallback else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(renameError)) + } + try renameWithoutExclusiveSupport(source, to: destination) } - /// See ``FileSystem/copy(from:to:recursive:)``. - public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) - async throws - { - let configuration = try requireConfiguration() - let source = try noFollowURL(for: sourcePath, configuration: configuration) - let destination = try creationURL(for: destinationPath, configuration: configuration) - - let sourceInfo = try stat(path: sourcePath, configuration: configuration) - if sourceInfo.kind == .directory, !recursive { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) + private func renameWithoutExclusiveSupport(_ source: Entry, to destination: Entry) throws { + do { + _ = try metadata(destination) + } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { + // Reuse the resolved descriptors. Only no-clobber atomicity is relaxed, not containment. + try check(renameat(source.parent.value, source.name, destination.parent.value, destination.name)) + return } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EEXIST)) + } + + private static func renameExclusively( + sourceFD: Int32, sourceName: String, destinationFD: Int32, destinationName: String + ) -> Int32 { + #if canImport(Darwin) + let result = renameatx_np(sourceFD, sourceName, destinationFD, destinationName, UInt32(RENAME_EXCL)) + let renameError = errno + return result == 0 ? 0 : renameError + #else + // Some Swift Glibc modules omit renameat2. Resolve the libc entry point without an + // architecture-specific syscall number; a missing symbol is treated like missing kernel support. + typealias RenameAt2 = @convention(c) ( + Int32, UnsafePointer, Int32, UnsafePointer, UInt32 + ) -> Int32 + guard let symbol = dlsym(nil, "renameat2") else { + return ENOSYS + } + let rename = unsafeBitCast(symbol, to: RenameAt2.self) + return sourceName.withCString { source in + destinationName.withCString { destination in + let result = rename(sourceFD, source, destinationFD, destination, 1) // RENAME_NOREPLACE. + let renameError = errno + return result == 0 ? 0 : renameError + } + } + #endif + } - let parent = destination.deletingLastPathComponent() - try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) - - if sourceInfo.kind == .directory { - try fileManager.copyItem(at: source, to: destination) - } else { - if fileManager.fileExists(atPath: destination.path) { - try fileManager.removeItem(at: destination) + private func copyEntry(_ source: Entry, to destination: Entry, recursive: Bool) throws { + let info = try metadata(source) + switch info.st_mode & S_IFMT { + case S_IFDIR: + guard recursive else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) + } + let input = try openEntry(source, flags: O_RDONLY | O_DIRECTORY) + try check(mkdirat(destination.parent.value, destination.name, 0o755)) + let output = try openEntry(destination, flags: O_RDONLY | O_DIRECTORY) + for name in try names(in: input) { + try copyEntry( + Entry(parent: input, name: name, path: source.path.appending(name)), + to: Entry(parent: output, name: name, path: destination.path.appending(name)), recursive: true) + } + try check(fchmod(output.value, info.st_mode & 0o7777)) + case S_IFLNK: + let target = try linkTarget(source) + try removeEntry(destination, recursive: false) + try check(symlinkat(target, destination.parent.value, destination.name)) + case S_IFREG: + let input = try openEntry(source, flags: O_RDONLY | O_NONBLOCK) + let handle = FileHandle(fileDescriptor: try duplicateDescriptor(input.value), closeOnDealloc: true) + defer { + try? handle.close() + } + try replace(destination) { output in + while let data = try handle.read(upToCount: 65536), !data.isEmpty { + try write(data, to: output) + } + try check(fchmod(output.value, info.st_mode & 0o7777)) } - try fileManager.copyItem(at: source, to: destination) + default: throw WorkspaceError.unsupported("copying special files is not supported") } } + /// See ``FileSystem/copy(from:to:recursive:)``. + /// + /// Copies contents and POSIX mode bits; extended attributes and ACLs are not copied. + public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + let config = try requireConfiguration() + let source = try resolve(sourcePath, followFinal: false, configuration: config) + let destination = try resolve(destinationPath, followFinal: false, createParents: true, configuration: config) + guard source.path != destination.path, + !(destination.path.string.hasPrefix(source.path.string + "/")) && !source.path.isRoot + else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EINVAL)) + } + try copyEntry(source, to: destination, recursive: recursive) + } + /// See ``FileSystem/createSymlink(path:target:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { _ = try WorkspacePath(validating: target, relativeTo: path.dirname) - let configuration = try requireConfiguration() - let url = try creationURL(for: path, configuration: configuration) - let parent = url.deletingLastPathComponent() - try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) - try fileManager.createSymbolicLink(atPath: url.path, withDestinationPath: target) + let entry = try resolve(path, followFinal: false, createParents: true) + try check(symlinkat(target, entry.parent.value, entry.name)) } /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { - let configuration = try requireConfiguration() - let linkURL = try creationURL(for: path, configuration: configuration) - let targetURL = try existingURL(for: target, configuration: configuration) - - let parent = linkURL.deletingLastPathComponent() - try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) - try fileManager.linkItem(at: targetURL, to: linkURL) + let config = try requireConfiguration() + let source = try resolve(target, followFinal: true, configuration: config) + let destination = try resolve(path, followFinal: false, createParents: true, configuration: config) + // No AT_SYMLINK_FOLLOW: a concurrent final-component swap must not dereference a link. + try check(linkat(source.parent.value, source.name, destination.parent.value, destination.name, 0)) } /// See ``FileSystem/readSymlink(path:)``. - /// - /// Uses `lstat` semantics so the target string of a link pointing outside the root can still - /// be read without dereferencing it. public func readSymlink(path: WorkspacePath) async throws -> String { - let configuration = try requireConfiguration() - let url = try noFollowURL(for: path, configuration: configuration) - return try fileManager.destinationOfSymbolicLink(atPath: url.path) + try linkTarget(resolve(path, followFinal: false)) } /// See ``FileSystem/setPermissions(path:permissions:)``. + /// + /// Changes the resolved target without following a concurrently substituted final symlink. + /// On Linux before 6.6, glibc emulates `fchmodat(AT_SYMLINK_NOFOLLOW)` via `/proc/self/fd`; + /// chmod therefore fails if `/proc` is not mounted. public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { - let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) - try fileManager.setAttributes([.posixPermissions: Int(permissions.rawValue)], ofItemAtPath: url.path) + let entry = try resolve(path, followFinal: true) + try check(fchmodat(entry.parent.value, entry.name, mode_t(permissions.rawValue), AT_SYMLINK_NOFOLLOW)) } /// See ``FileSystem/resolveRealPath(path:)``. + /// + /// Returns the resolved virtual path relative to the retained root, never a fallback `/`. public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { - let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) - let resolved = url.resolvingSymlinksInPath().standardizedFileURL - try ensureInsideRoot(resolved, configuration: configuration) - return virtualPath(from: resolved, configuration: configuration) + let entry = try resolve(path, followFinal: true) + _ = try metadata(entry) + return entry.path } /// See ``FileSystem/exists(path:)``. /// - /// Uses `lstat` semantics behind the same containment check as other entry operations: a - /// symlink exists even when dangling, and its target is never resolved, so this cannot be - /// used to probe paths outside the root. + /// Uses lstat semantics, so a final dangling symlink still exists. public func exists(path: WorkspacePath) async -> Bool { - if path.string.contains("\u{0}") { - return false - } - do { - let configuration = try requireConfiguration() - let url = try noFollowURL(for: path, configuration: configuration) - return (try? fileManager.attributesOfItem(atPath: url.path)) != nil - } catch { - return false - } + (try? metadata(resolve(path, followFinal: false))) != nil + } + + /// See ``FileSystem/capabilities()``. + public func capabilities() async -> FileSystemFeatures { + [.symlinks, .hardLinks, .permissions, .realPathResolution] } /// See ``FileSystem/glob(pattern:currentDirectory:)``. public func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { - let normalizedPattern = try WorkspacePath(validating: pattern, relativeTo: currentDirectory) - if !WorkspacePath.containsGlob(normalizedPattern.string) { - return await exists(path: normalizedPattern) ? [normalizedPattern] : [] + let pattern = try WorkspacePath(validating: pattern, relativeTo: currentDirectory) + if !WorkspacePath.containsGlob(pattern.string) { + return await exists(path: pattern) ? [pattern] : [] } - - let regex = try NSRegularExpression(pattern: WorkspacePath.globToRegex(normalizedPattern.string)) - let allPaths = try allVirtualPaths() - - let matches = allPaths.filter { path in - let range = NSRange(path.string.startIndex.. [WorkspacePath] { - let configuration = try requireConfiguration() - let root = configuration.rootURL + let regex = try NSRegularExpression(pattern: WorkspacePath.globToRegex(pattern.string)) var paths: [WorkspacePath] = [.root] - - guard let enumerator = fileManager.enumerator(at: root, includingPropertiesForKeys: nil) else { - return paths - } - - for case let url as URL in enumerator { - paths.append(virtualPath(from: url, configuration: configuration)) - } - - return paths - } - - private func existingOrPotentialURL( - for virtualPath: WorkspacePath, - configuration: ConfigurationSnapshot - ) throws -> URL { - let root = configuration.rootURL - if virtualPath.isRoot { - return root - } - - let relative = String(virtualPath.string.dropFirst()) - return root.appendingPathComponent(relative) - } - - private func existingURL( - for virtualPath: WorkspacePath, - configuration: ConfigurationSnapshot - ) throws -> URL { - let url = try existingOrPotentialURL(for: virtualPath, configuration: configuration) - try ensureInsideRoot(url, configuration: configuration) - return url - } - - private func creationURL( - for virtualPath: WorkspacePath, - configuration: ConfigurationSnapshot - ) throws -> URL { - let url = try existingOrPotentialURL(for: virtualPath, configuration: configuration) - let parent = url.deletingLastPathComponent() - try ensureInsideRoot(parent, configuration: configuration) - try ensureDestinationSymlinkStaysInsideRoot(url, configuration: configuration) - return url - } - - /// Returns a jailed URL for operations on the entry itself (`lstat` semantics): the parent - /// chain is resolved and containment-checked, but a symlink at the final component is not - /// followed. This lets callers stat, remove, or move a symlink whose target lies outside the - /// root without ever dereferencing it. - private func noFollowURL( - for virtualPath: WorkspacePath, - configuration: ConfigurationSnapshot - ) throws -> URL { - let url = try existingOrPotentialURL(for: virtualPath, configuration: configuration) - if virtualPath.isRoot { - try ensureInsideRoot(url, configuration: configuration) - return url - } - try ensureInsideRoot(url.deletingLastPathComponent(), configuration: configuration) - return url - } - - /// The parent containment check never resolves the destination's final component, so a - /// pre-existing symlink at the destination could redirect follow-on opens (for example an - /// append) outside the configured root. Reject destinations whose final component is a - /// symlink resolving outside the root. - private func ensureDestinationSymlinkStaysInsideRoot( - _ url: URL, - configuration: ConfigurationSnapshot - ) throws { - let attributes = try? fileManager.attributesOfItem(atPath: url.path) - guard attributes?[.type] as? FileAttributeType == .typeSymbolicLink else { - return - } - try ensureInsideRoot(url, configuration: configuration) - } - - private func ensureInsideRoot(_ url: URL, configuration: ConfigurationSnapshot) throws { - let resolved = url.resolvingSymlinksInPath().standardizedFileURL.path - let root = configuration.resolvedRootPath - guard resolved == root || resolved.hasPrefix(root + "/") else { - throw WorkspaceError.invalidPath(virtualPath(from: url, configuration: configuration).description) - } - } - - private func virtualPath(from physicalURL: URL, configuration: ConfigurationSnapshot) -> WorkspacePath { - let rootPath = configuration.rootURL.path - let path = physicalURL.standardizedFileURL.path - - if path == rootPath { - return .root - } - - guard path.hasPrefix(rootPath) else { - return .root - } - - let start = path.index(path.startIndex, offsetBy: rootPath.count) - let suffix = String(path[start...]).trimmingCharacters(in: CharacterSet(charactersIn: "/")) - if suffix.isEmpty { - return .root - } - return WorkspacePath(unchecked: "/" + suffix) - } - - private func requireConfiguration() throws -> ConfigurationSnapshot { - stateLock.lock() - defer { stateLock.unlock() } - guard let rootURL, let resolvedRootPath else { - throw WorkspaceError.unsupported("local filesystem requires a root") - } - return ConfigurationSnapshot(rootURL: rootURL, resolvedRootPath: resolvedRootPath) - } - - private func stat(path: WorkspacePath, configuration: ConfigurationSnapshot) throws -> FileInfo { - let url = try noFollowURL(for: path, configuration: configuration) - let attributes = try fileManager.attributesOfItem(atPath: url.path) - - let fileType = attributes[.type] as? FileAttributeType - let isDirectory = fileType == .typeDirectory - let isSymbolicLink = fileType == .typeSymbolicLink - let size = (attributes[.size] as? NSNumber)?.uint64Value ?? 0 - let permissionBits = (attributes[.posixPermissions] as? NSNumber)?.intValue ?? 0 - let modificationDate = attributes[.modificationDate] as? Date - - let kind: FileTree.Kind - if isSymbolicLink { - kind = .symlink - } else if isDirectory { - kind = .directory - } else { - kind = .file + var queue: [WorkspacePath] = [.root] + while !queue.isEmpty { + let directory = queue.removeFirst() + let entries: [DirectoryEntry] + do { + entries = try await listDirectory(path: directory) + } catch let error as NSError + where !directory.isRoot + && error.domain == NSPOSIXErrorDomain + && [Int(EACCES), Int(ENOENT), Int(ENOTDIR)].contains(error.code) + { + continue + } + for entry in entries { + paths.append(entry.info.path) + if entry.info.kind == .directory { + queue.append(entry.info.path) + } + } } - - return FileInfo( - path: path, - kind: kind, - size: size, - permissions: POSIXPermissions(permissionBits), - modificationDate: modificationDate - ) + return paths.filter { + regex.firstMatch(in: $0.string, range: NSRange($0.string.startIndex..., in: $0.string)) != nil + }.sorted() } } #if os(macOS) extension LocalFileSystem: FileSystemChangeSource { + /// See ``FileSystemChangeSource/changes()``. public func changes() async throws -> AsyncStream { let root = try requireConfiguration().rootURL return try FSEventsChangeStream.observe(root) @@ -593,6 +775,7 @@ enum FSEventsChangeStream { } #elseif os(Linux) extension LocalFileSystem: FileSystemChangeSource { + /// See ``FileSystemChangeSource/changes()``. public func changes() async throws -> AsyncStream { try DirectoryChangeStream.observe(requireConfiguration().rootURL, recursive: true) } diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index 9c86ef6..2d9aee6 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -96,7 +96,8 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { - guard let info = await mergedEntry(path) else { + let resolved = try await resolveMergedPath(path, followFinal: false) + guard let info = await mergedEntry(resolved) else { throw posixError(ENOENT) } var adjusted = info @@ -106,6 +107,8 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/listDirectory(path:)``. public func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + let originalPath = path + let path = try await resolveMergedPath(path) let upperInfo = await upperEntry(path) let lowerInfo = await lowerVisibleEntry(path) guard upperInfo != nil || lowerInfo != nil else { @@ -132,12 +135,16 @@ public actor OverlayFileSystem: FileSystem { byName[entry.name] = entry } } - return byName.values.sorted { $0.name < $1.name } + return byName.values.map { entry in + var info = entry.info + info.path = originalPath.appending(entry.name) + return DirectoryEntry(name: entry.name, info: info) + }.sorted { $0.name < $1.name } } /// See ``FileSystem/readFile(path:)``. public func readFile(path: WorkspacePath) async throws -> Data { - let target = try await resolveMergedFinalSymlink(path) + let target = try await resolveMergedPath(path) if await upperEntry(target) != nil { return try await upper.readFile(path: target) } @@ -152,6 +159,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/readSymlink(path:)``. public func readSymlink(path: WorkspacePath) async throws -> String { + let path = try await resolveMergedPath(path, followFinal: false) if await upperEntry(path) != nil { return try await upper.readSymlink(path: path) } @@ -166,7 +174,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/resolveRealPath(path:)``. public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { - let target = try await resolveMergedFinalSymlink(path) + let target = try await resolveMergedPath(path) guard await mergedEntry(target) != nil else { throw posixError(ENOENT) } @@ -175,7 +183,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/exists(path:)``. public func exists(path: WorkspacePath) async -> Bool { - guard let target = try? await resolveMergedFinalSymlink(path) else { + guard let target = try? await resolveMergedPath(path) else { return false } return await mergedEntry(target) != nil @@ -208,7 +216,7 @@ public actor OverlayFileSystem: FileSystem { guard !path.isRoot else { throw posixError(EISDIR) } - let target = try await resolveMergedFinalSymlink(path) + let target = try await resolveMergedPath(path) if await upperEntry(target) != nil { try await upper.writeFile(path: target, data: data, append: append) return @@ -234,6 +242,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/createFile(path:data:)``. public func createFile(path: WorkspacePath, data: Data) async throws { + let path = try await resolveMergedPath(path, followFinal: false) guard await mergedEntry(path) == nil else { throw posixError(EEXIST) } @@ -243,6 +252,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/createDirectory(path:recursive:)``. public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { + let path = try await resolveMergedPath(path, allowMissingAncestors: recursive) if path.isRoot { return } @@ -273,6 +283,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/remove(path:recursive:)``. public func remove(path: WorkspacePath, recursive: Bool) async throws { + let path = try await resolveMergedPath(path, followFinal: false) guard !path.isRoot else { throw posixError(EPERM) } @@ -301,6 +312,8 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/move(from:to:)``. public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { + let sourcePath = try await resolveMergedPath(sourcePath, followFinal: false) + let destinationPath = try await resolveMergedPath(destinationPath, followFinal: false) if sourcePath == destinationPath { return } @@ -327,6 +340,8 @@ public actor OverlayFileSystem: FileSystem { public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + let sourcePath = try await resolveMergedPath(sourcePath, followFinal: false) + let destinationPath = try await resolveMergedPath(destinationPath, followFinal: false) guard let sourceInfo = await mergedEntry(sourcePath) else { throw posixError(ENOENT) } @@ -344,6 +359,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/createSymlink(path:target:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { + let path = try await resolveMergedPath(path, followFinal: false) _ = try WorkspacePath(validating: target, relativeTo: path.dirname) guard await mergedEntry(path) == nil else { throw posixError(EEXIST) @@ -354,6 +370,8 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { + let path = try await resolveMergedPath(path, followFinal: false) + let target = try await resolveMergedPath(target) guard let targetInfo = await mergedEntry(target) else { throw posixError(ENOENT) } @@ -373,6 +391,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/setPermissions(path:permissions:)``. public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { + let path = try await resolveMergedPath(path) if await upperEntry(path) == nil { guard let info = await lowerVisibleEntry(path) else { throw posixError(ENOENT) @@ -414,27 +433,57 @@ public actor OverlayFileSystem: FileSystem { return await lowerVisibleEntry(path) } - /// Follows a chain of symlinks at the final path component through the merged view, so a - /// link in one layer can point at an entry in the other. - private func resolveMergedFinalSymlink(_ path: WorkspacePath) async throws -> WorkspacePath { - var current = path - var depth = 0 - while depth <= 64 { - guard let info = await mergedEntry(current), info.kind == .symlink else { - return current + /// Resolve each component in the merged namespace before consulting a backing filesystem. + /// A lower-layer ancestor link must never let that backend bypass cuts or upper entries. + private func resolveMergedPath( + _ path: WorkspacePath, followFinal: Bool = true, allowMissingAncestors: Bool = false + ) async throws -> WorkspacePath { + var current = WorkspacePath.root + var pending = path.components + var hops = 0 + while !pending.isEmpty { + let component = pending.removeFirst() + if component == "." { + continue } - let target: String - if await upperEntry(current) != nil { - target = try await upper.readSymlink(path: current) - } else if let lower { - target = try await lower.readSymlink(path: current) - } else { - return current + if component == ".." { + current = current.dirname + continue + } + let next = current.appending(component) + let final = pending.isEmpty + guard let info = await mergedEntry(next) else { + if final || allowMissingAncestors { + current = next + continue + } + throw posixError(ENOENT) + } + if info.kind == .symlink && (!final || followFinal) { + hops += 1 + guard hops <= 40 else { + throw posixError(ELOOP) + } + let target: String + if await upperEntry(next) != nil { + target = try await upper.readSymlink(path: next) + } else if let lower { + target = try await lower.readSymlink(path: next) + } else { + throw posixError(ENOENT) + } + if target.hasPrefix("/") { + current = .root + } + pending = target.split(separator: "/").map(String.init) + pending + continue + } + if !final && info.kind != .directory { + throw posixError(ENOTDIR) } - current = WorkspacePath(normalizing: target, relativeTo: current.dirname) - depth += 1 + current = next } - throw posixError(ELOOP) + return current } /// Materializes every merged-visible ancestor directory of `path` into the upper layer, diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 363d9c0..b78bea2 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -113,6 +113,8 @@ public protocol PermissionAuthorizing: Sendable { } /// An actor-backed authorization policy with built-in support for session-scoped approvals. +/// When used by `AuthorizedFileSystem`, a symlinked path that resolves to a different target +/// produces a second authorization request for that target, with its own cache and audit entry. public actor PermissionAuthorizer: PermissionAuthorizing { /// The asynchronous callback used to evaluate requests. public typealias Handler = @Sendable (PermissionRequest) async -> PermissionDecision @@ -200,6 +202,8 @@ public struct PermissionRule: Sendable, Codable, Equatable { } /// A deterministic first-match path-prefix authorization policy. +/// Deny rules match any endpoint; allow rules must cover every endpoint. +/// Glob's `currentDirectory` is context rather than an endpoint and is excluded from deny matching. public actor RuleBasedPermissionAuthorizer: PermissionAuthorizing { private let rules: [PermissionRule] private let defaultEffect: PermissionRule.Effect @@ -218,7 +222,7 @@ public actor RuleBasedPermissionAuthorizer: PermissionAuthorizing { public func authorize(_ request: PermissionRequest) async -> PermissionDecision { for (index, rule) in rules.enumerated() - where rule.operations.contains(request.operation) && Self.matchesPaths(request, prefix: rule.pathPrefix) { + where rule.operations.contains(request.operation) && Self.matchesPaths(request, rule: rule) { return decision(for: rule.effect, request: request, source: .rule(index)) } return decision(for: defaultEffect, request: request, source: .defaultRule) @@ -227,11 +231,19 @@ public actor RuleBasedPermissionAuthorizer: PermissionAuthorizing { public func auditLog() -> [PermissionAuditRecord] { auditRecords } public func clearAuditLog() { auditRecords.removeAll(keepingCapacity: true) } - private static func matchesPaths(_ request: PermissionRequest, prefix: WorkspacePath) -> Bool { - let paths = [request.path, request.sourcePath, request.destinationPath].compactMap { $0 } + private static func matchesPaths(_ request: PermissionRequest, rule: PermissionRule) -> Bool { + var endpoints = [request.path, request.sourcePath, request.destinationPath] + if request.operation == .glob, case .deny = rule.effect { + endpoints = [request.path, request.sourcePath] + } + let paths = endpoints.compactMap { $0 } guard !paths.isEmpty else { return false } - return paths.allSatisfy { path in - prefix.isRoot || path == prefix || path.string.hasPrefix(prefix.string + "/") + let matches: (WorkspacePath) -> Bool = { path in + rule.pathPrefix.isRoot || path == rule.pathPrefix || path.string.hasPrefix(rule.pathPrefix.string + "/") + } + switch rule.effect { + case .allow: return paths.allSatisfy(matches) + case .deny: return paths.contains(where: matches) } } @@ -260,7 +272,15 @@ public actor RuleBasedPermissionAuthorizer: PermissionAuthorizing { } } -/// A filesystem wrapper that authorizes each operation before forwarding it to another filesystem. +/// Authorizes both the requested spelling and the resolved paths touched by an operation. +/// A symlinked path produces a second authorization request when its resolved target differs; +/// both requests must be allowed. Authorizers see both requests separately for caching and auditing. +/// Entry operations resolve ancestors but retain the final link itself (lstat semantics). +/// Resolution and execution are separate protocol calls: concurrent namespace changes can still +/// race authorization. This wrapper is a policy safety layer, not a hostile-process sandbox. +/// Backends advertising symlinks must support real-path resolution; unsupported resolution fails closed. +/// Creation through an unresolved dangling link also fails closed. Existence checks conservatively +/// authorize the target because some backends implement them with dereferencing semantics. public final class AuthorizedFileSystem: FileSystem, Sendable { private let base: any FileSystem private let authorizer: any PermissionAuthorizing @@ -309,7 +329,7 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { /// See ``FileSystem/createFile(path:data:)``. public func createFile(path: WorkspacePath, data: Data) async throws { - try await authorize(.init(operation: .writeFile, path: path)) + try await authorize(.init(operation: .writeFile, path: path), entryOnly: true) try await base.createFile(path: path, data: data) } @@ -408,10 +428,92 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { return try await base.glob(pattern: normalizedPattern.description, currentDirectory: currentDirectory) } - private func authorize(_ request: PermissionRequest) async throws { + private func checkDecision(_ request: PermissionRequest) async throws { let decision = await authorizer.authorize(request) - if case let .deny(message) = decision { + if case .deny(let message) = decision { throw WorkspaceError.accessDenied(operation: request.operation.rawValue, message: message) } } + + private func resolvedPath(_ path: WorkspacePath, followFinal: Bool) async throws -> WorkspacePath { + if !followFinal, !path.isRoot { + return try await resolvedPath(path.dirname, followFinal: true).appending(path.basename) + } + do { + return try await base.resolveRealPath(path: path) + } catch { + let failure = error as NSError + guard failure.domain == NSPOSIXErrorDomain, failure.code == Int(ENOENT) else { + throw error + } + // A dangling link must fail closed; synthetic mount parents and absent entries can + // use their resolved parent and let the base operation decide whether they exist. + if let info = try? await base.stat(path: path), info.kind == .symlink { + throw error + } + if path.isRoot { + return .root + } + return try await resolvedPath(path.dirname, followFinal: true).appending(path.basename) + } + } + + private func authorize(_ request: PermissionRequest, entryOnly: Bool = false) async throws { + try await checkDecision(request) + let features = await base.capabilities() + guard features.contains(.realPathResolution) || features.contains(.symlinks) else { + return + } + var resolved = request + switch request.operation { + case .stat, .remove, .readSymlink: + if let path = request.path { + resolved.path = try await resolvedPath(path, followFinal: false) + } + case .exists: + if let path = request.path { + resolved.path = try await resolvedPath(path, followFinal: true) + } + case .move, .copy: + if let path = request.sourcePath { + resolved.sourcePath = try await resolvedPath(path, followFinal: false) + } + if let path = request.destinationPath { + resolved.destinationPath = try await resolvedPath(path, followFinal: false) + } + case .createSymlink, .createHardLink: + if let path = request.path { + resolved.path = try await resolvedPath(path, followFinal: false) + } + if let path = request.destinationPath { + resolved.destinationPath = try await resolvedPath(path, followFinal: true) + } + case .glob: + // Glob enumerates entries without dereferencing matched final links. + // Only its non-pattern prefix needs resolution. + if let path = request.path { + var prefix = WorkspacePath.root + var suffix: [String] = [] + for component in path.components { + if !suffix.isEmpty || WorkspacePath.containsGlob(component) { + suffix.append(component) + } else { + prefix = prefix.appending(component) + } + } + var target = try await resolvedPath(prefix, followFinal: true) + for component in suffix { + target = target.appending(component) + } + resolved.path = target + } + default: + if let path = request.path { + resolved.path = try await resolvedPath(path, followFinal: !entryOnly) + } + } + if resolved != request { + try await checkDecision(resolved) + } + } } diff --git a/Tests/WorkspaceTests/FilesystemTests.swift b/Tests/WorkspaceTests/FilesystemTests.swift index 6b850a1..e5c723b 100644 --- a/Tests/WorkspaceTests/FilesystemTests.swift +++ b/Tests/WorkspaceTests/FilesystemTests.swift @@ -916,12 +916,14 @@ struct FilesystemTests { .readFile, .readSymlink, .setPermissions, - .resolveRealPath, + .resolveRealPath, // lexical link + .resolveRealPath, // resolved target .exists, - .glob, + .glob, // pattern .remove, ]) + #expect(requests.filter { $0.operation == .resolveRealPath }.map(\.path) == ["/links/link.txt", "/note.txt"]) let symlinkRequest = try #require(requests.first { $0.operation == .createSymlink }) #expect(symlinkRequest.path == "/links/link.txt") #expect(symlinkRequest.destinationPath == "/note.txt") diff --git a/Tests/WorkspaceTests/PathResolutionTests.swift b/Tests/WorkspaceTests/PathResolutionTests.swift new file mode 100644 index 0000000..f434834 --- /dev/null +++ b/Tests/WorkspaceTests/PathResolutionTests.swift @@ -0,0 +1,569 @@ +import Foundation +import Testing + +@testable import Workspace + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +@Suite("Path resolution and authorization") +struct PathResolutionTests { + @Test + func authorizationChecksResolvedTargets() async throws { + let root = try TestSupport.temporaryDirectory("AuthorizationPaths") + defer { + TestSupport.remove(root) + } + let base = try LocalFileSystem(root: root) + try await base.createDirectory(path: "/public", recursive: true) + try await base.createDirectory(path: "/private", recursive: true) + try await base.writeFile(path: "/private/secret", data: Data("secret".utf8), append: false) + try await base.createSymlink(path: "/public/link", target: "../private/secret") + try await base.createSymlink(path: "/public/alias", target: "../private") + let operations: Set = [ + .readFile, .stat, .exists, .writeFile, .setPermissions, .remove, .readSymlink, + ] + let fs = AuthorizedFileSystem( + base: base, + authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: operations, pathPrefix: "/public", effect: .allow) + ])) + for path: WorkspacePath in ["/private/secret", "/public/link", "/public/alias/secret"] { + await #expect(throws: WorkspaceError.self) { + _ = try await fs.readFile(path: path) + } + } + await #expect(throws: WorkspaceError.self) { + _ = try await fs.stat(path: "/public/alias/secret") + } + #expect(!(await fs.exists(path: "/public/alias/secret"))) + for append in [false, true] { + await #expect(throws: WorkspaceError.self) { + try await fs.writeFile(path: "/public/link", data: Data("leak".utf8), append: append) + } + } + await #expect(throws: WorkspaceError.self) { + try await fs.createFile(path: "/public/alias/new", data: Data()) + } + await #expect(throws: WorkspaceError.self) { + try await fs.setPermissions(path: "/public/link", permissions: POSIXPermissions(0o777)) + } + #expect(try await fs.stat(path: "/public/link").kind == .symlink) + #expect(try await fs.readSymlink(path: "/public/link") == "../private/secret") + try await fs.remove(path: "/public/link", recursive: false) + #expect(try await base.readFile(path: "/private/secret") == Data("secret".utf8)) + } + + @Test + func denyRulesMatchEitherEndpoint() async throws { + for operation: PermissionOperation in [.copy, .move, .createHardLink] { + for reverse in [false, true] { + let base = InMemoryFileSystem() + try await base.createDirectory(path: "/private", recursive: true) + try await base.createDirectory(path: "/public", recursive: true) + let source: WorkspacePath = reverse ? "/public/source" : "/private/source" + let destination: WorkspacePath = reverse ? "/private/destination" : "/public/destination" + try await base.writeFile(path: source, data: Data("secret".utf8), append: false) + let fs = AuthorizedFileSystem( + base: base, + authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [operation], pathPrefix: "/private", effect: .deny(message: "private")), + .init(operations: [operation], pathPrefix: "/", effect: .allow), + ])) + await #expect(throws: WorkspaceError.self) { + switch operation { + case .copy: try await fs.copy(from: source, to: destination, recursive: false) + case .move: try await fs.move(from: source, to: destination) + default: try await fs.createHardLink(path: destination, target: source) + } + } + #expect(!(await base.exists(path: destination))) + #expect(await base.exists(path: source)) + } + } + } + + @Test + func rootDescriptorSurvivesRootPathReplacement() async throws { + let container = try TestSupport.temporaryDirectory("PinnedRoot") + defer { + TestSupport.remove(container) + } + let root = container.appendingPathComponent("root") + let outside = container.appendingPathComponent("outside") + let fs = try LocalFileSystem(root: root) + try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true) + try await fs.writeFile(path: "/file", data: Data("inside".utf8), append: false) + try Data("outside".utf8).write(to: outside.appendingPathComponent("file")) + try FileManager.default.moveItem(at: root, to: container.appendingPathComponent("original")) + try FileManager.default.createSymbolicLink(at: root, withDestinationURL: outside) + #expect(try await fs.readFile(path: "/file") == Data("inside".utf8)) + try await fs.writeFile(path: "/file", data: Data("!".utf8), append: true) + #expect(try Data(contentsOf: outside.appendingPathComponent("file")) == Data("outside".utf8)) + } + + @Test + func realPathsUseCanonicalRoot() async throws { + let container = try TestSupport.temporaryDirectory("RootAlias") + defer { + TestSupport.remove(container) + } + let physical = container.appendingPathComponent("physical") + let original = try LocalFileSystem(root: physical) + try await original.writeFile(path: "/file", data: Data(), append: false) + let alias = container.appendingPathComponent("alias") + try FileManager.default.createSymbolicLink(at: alias, withDestinationURL: physical) + let fs = try LocalFileSystem(root: alias) + #expect(try await fs.resolveRealPath(path: "/file") == "/file") + } + + @Test + func chunksReadOnlyWhenRequestedAndKeepTheirOpenFile() async throws { + let root = try TestSupport.temporaryDirectory("DemandChunks") + defer { + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + try await fs.writeFile(path: "/file", data: Data("aaaabbbbcccc".utf8), append: false) + let stream = try await fs.readFileChunks(path: "/file", chunkSize: 4) + var iterator = stream.makeAsyncIterator() + #expect(try await iterator.next() == Data("aaaa".utf8)) + // A paused consumer must not cause subsequent bytes to be read or buffered. + try await Task.sleep(for: .milliseconds(100)) + let handle = try FileHandle(forWritingTo: root.appendingPathComponent("file")) + try handle.seek(toOffset: 4) + try handle.write(contentsOf: Data("BBBBCCCC".utf8)) + try handle.close() + try await fs.move(from: "/file", to: "/old") + try await fs.writeFile(path: "/file", data: Data("replacement".utf8), append: false) + #expect(try await iterator.next() == Data("BBBB".utf8)) + #expect(try await iterator.next() == Data("CCCC".utf8)) + #expect(try await iterator.next() == nil) + } + + @Test + func overlayResolvesAncestorsBeforeWhiteoutsAndCopyUp() async throws { + let base = InMemoryFileSystem() + try await base.createDirectory(path: "/dir", recursive: true) + try await base.writeFile(path: "/dir/file", data: Data("lower".utf8), append: false) + try await base.createSymlink(path: "/alias", target: "dir") + let fs = OverlayFileSystem(over: base) + try await fs.writeFile(path: "/alias/file", data: Data("upper".utf8), append: false) + #expect(try await fs.readFile(path: "/dir/file") == Data("upper".utf8)) + #expect(try await base.readFile(path: "/dir/file") == Data("lower".utf8)) + #expect(try await fs.resolveRealPath(path: "/alias/file") == "/dir/file") + #expect(try await fs.listDirectory(path: "/alias").map(\.name) == ["file"]) + try await fs.remove(path: "/dir", recursive: true) + await #expect(throws: (any Error).self) { + _ = try await fs.readFile(path: "/alias/file") + } + #expect(!(await fs.exists(path: "/alias/file"))) + await #expect(throws: (any Error).self) { + _ = try await fs.stat(path: "/alias/file") + } + await #expect(throws: (any Error).self) { + try await fs.createFile(path: "/alias/new", data: Data()) + } + } + + @Test + func swappedAncestorCannotReachOutsideForAnyLocalOperation() async throws { + let container = try TestSupport.temporaryDirectory("SwappedAncestor") + defer { + TestSupport.remove(container) + } + let root = container.appendingPathComponent("root") + let outside = container.appendingPathComponent("outside") + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/dir", recursive: false) + try await fs.writeFile(path: "/source", data: Data("inside".utf8), append: false) + try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true) + try Data("secret".utf8).write(to: outside.appendingPathComponent("file")) + try FileManager.default.removeItem(at: root.appendingPathComponent("dir")) + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("dir"), withDestinationURL: outside) + for operation in [ + "read", "range", "chunks", "stat", "list", "write", "append", "create", "mkdir", + "remove", "moveSource", "moveDestination", "copySource", "copyDestination", + "chmod", "symlink", "hardSource", "hardDestination", + ] { + await #expect(throws: WorkspaceError.self, "operation: \(operation)") { + switch operation { + case "read": _ = try await fs.readFile(path: "/dir/file") + case "range": _ = try await fs.readFile(path: "/dir/file", offset: 0, length: 1) + case "chunks": _ = try await fs.readFileChunks(path: "/dir/file", chunkSize: 1) + case "stat": _ = try await fs.stat(path: "/dir/file") + case "list": _ = try await fs.listDirectory(path: "/dir") + case "write", "append": + try await fs.writeFile(path: "/dir/file", data: Data("bad".utf8), append: operation == "append") + case "create": try await fs.createFile(path: "/dir/new", data: Data()) + case "mkdir": try await fs.createDirectory(path: "/dir/new/sub", recursive: true) + case "remove": try await fs.remove(path: "/dir/file", recursive: true) + case "moveSource": try await fs.move(from: "/dir/file", to: "/leak") + case "moveDestination": try await fs.move(from: "/source", to: "/dir/leak") + case "copySource": try await fs.copy(from: "/dir/file", to: "/leak", recursive: true) + case "copyDestination": try await fs.copy(from: "/source", to: "/dir/leak", recursive: true) + case "chmod": try await fs.setPermissions(path: "/dir/file", permissions: POSIXPermissions(0o777)) + case "symlink": try await fs.createSymlink(path: "/dir/leak", target: "file") + case "hardSource": try await fs.createHardLink(path: "/leak", target: "/dir/file") + default: try await fs.createHardLink(path: "/dir/leak", target: "/source") + } + } + } + #expect(!(await fs.exists(path: "/dir/file"))) + #expect(try FileManager.default.contentsOfDirectory(atPath: outside.path) == ["file"]) + #expect(try Data(contentsOf: outside.appendingPathComponent("file")) == Data("secret".utf8)) + #expect(try await fs.readFile(path: "/source") == Data("inside".utf8)) + } + + @Test + func overlayWhiteoutsBlockDiskAliasesAndRouteEntryMutations() async throws { + let root = try TestSupport.temporaryDirectory("OverlayAliases") + defer { + TestSupport.remove(root) + } + let base = try LocalFileSystem(root: root) + try await base.createDirectory(path: "/dir", recursive: false) + try await base.writeFile(path: "/dir/file", data: Data("lower".utf8), append: false) + try await base.createSymlink(path: "/alias", target: "dir") + let fs = OverlayFileSystem(over: base) + try await fs.remove(path: "/dir", recursive: true) + await #expect(throws: (any Error).self) { + _ = try await fs.readFile(path: "/alias/file") + } + #expect(!(await fs.exists(path: "/alias/file"))) + try await fs.createDirectory(path: "/dir", recursive: false) + #expect(try await fs.listDirectory(path: "/alias").isEmpty) + try await fs.createFile(path: "/alias/new", data: Data("upper".utf8)) + try await fs.createSymlink(path: "/second", target: "alias") + try await fs.copy(from: "/second/new", to: "/alias/copied", recursive: false) + try await fs.move(from: "/alias/copied", to: "/second/moved") + try await fs.createHardLink(path: "/alias/hard", target: "/second/new") + try await fs.setPermissions(path: "/second/new", permissions: POSIXPermissions(0o600)) + #expect(try await fs.stat(path: "/dir/new").permissions == POSIXPermissions(0o600)) + #expect(try await fs.readFile(path: "/dir/hard") == Data("upper".utf8)) + #expect(try await fs.resolveRealPath(path: "/second/moved") == "/dir/moved") + try await fs.remove(path: "/second/moved", recursive: false) + #expect(!(await fs.exists(path: "/dir/moved"))) + #expect(try await base.readFile(path: "/dir/file") == Data("lower".utf8)) + #expect(try await base.listDirectory(path: "/dir").map(\.name) == ["file"]) + } + + @Test + func allowRulesStillRequireBothEndpoints() async throws { + let authorizer = RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.copy, .move, .createHardLink], pathPrefix: "/public", effect: .allow) + ]) + for operation: PermissionOperation in [.copy, .move, .createHardLink] { + for (source, destination): (WorkspacePath, WorkspacePath) in [ + ("/public/file", "/private/file"), ("/private/file", "/public/file"), + ("/publicity/file", "/public/file"), + ] { + let decision = await authorizer.authorize( + .init(operation: operation, sourcePath: source, destinationPath: destination)) + guard case .deny = decision else { + Issue.record("allow rule matched only one endpoint") + continue + } + } + } + } + + @Test + func localSymlinkWalkingSupportsInternalAliasesAndBoundsLoops() async throws { + let root = try TestSupport.temporaryDirectory("LocalLinks") + defer { + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/dir/sub", recursive: true) + try await fs.writeFile(path: "/dir/file", data: Data("inside".utf8), append: false) + try await fs.createSymlink(path: "/alias", target: "dir/sub") + try await fs.createSymlink(path: "/relative", target: "alias/../file") + try await fs.createSymlink(path: "/absolute", target: root.appendingPathComponent("dir/file").path) + #expect(try await fs.readFile(path: "/relative") == Data("inside".utf8)) + #expect(try await fs.resolveRealPath(path: "/absolute") == "/dir/file") + try await fs.createSymlink(path: "/cycle", target: "cycle") + await #expect(throws: (any Error).self) { + _ = try await fs.readFile(path: "/cycle") + } + try await fs.createSymlink(path: "/prefix", target: root.path + "-other/file") + await #expect(throws: WorkspaceError.self) { + _ = try await fs.resolveRealPath(path: "/prefix") + } + } + + // Root bypasses access checks, so these cases must run as an ordinary user to prove recovery. + @Test(.enabled(if: geteuid() != 0, "Root bypasses permission checks"), arguments: [false, true]) + func chmodRestoresInaccessibleEntries(directory: Bool) async throws { + let root = try TestSupport.temporaryDirectory("ChmodRecovery") + let entry = root.appendingPathComponent("entry") + defer { + _ = chmod(entry.path, 0o755) + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + if directory { + try await fs.createDirectory(path: "/entry", recursive: false) + } else { + try await fs.createFile(path: "/entry", data: Data("content".utf8)) + } + for mode in [0o000, 0o200] { + try await fs.setPermissions(path: "/entry", permissions: POSIXPermissions(mode)) + #expect(try await fs.stat(path: "/entry").permissions == POSIXPermissions(mode)) + try await fs.setPermissions(path: "/entry", permissions: POSIXPermissions(0o644)) + #expect(try await fs.stat(path: "/entry").permissions == POSIXPermissions(0o644)) + } + } + + @Test + func replacementPreservesExecutableMode() async throws { + let root = try TestSupport.temporaryDirectory("ReplacementMode") + defer { + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + try await fs.createFile(path: "/script", data: Data("old".utf8)) + try await fs.setPermissions(path: "/script", permissions: POSIXPermissions(0o755)) + try await fs.writeFile(path: "/script", data: Data("new".utf8), append: false) + #expect(try await fs.stat(path: "/script").permissions == POSIXPermissions(0o755)) + #expect(try await fs.readFile(path: "/script") == Data("new".utf8)) + } + + @Test + func globAuthorizesThePatternWithoutDereferencingMatchesOrDenyingUnusedCWD() async throws { + let root = try TestSupport.temporaryDirectory("GlobAuthorization") + defer { + TestSupport.remove(root) + } + let base = try LocalFileSystem(root: root) + try await base.createDirectory(path: "/public", recursive: false) + try await base.createDirectory(path: "/private", recursive: false) + try await base.createFile(path: "/public/file", data: Data()) + try await base.createSymlink(path: "/public/dangling", target: "missing") + try await base.createSymlink(path: "/public/external", target: root.path + "-outside/file") + let authorizer = PermissionAuthorizer { + _ in .allow + } + let fs = AuthorizedFileSystem(base: base, authorizer: authorizer) + let expected: [WorkspacePath] = ["/public/dangling", "/public/external", "/public/file"] + #expect(try await fs.glob(pattern: "/public/*", currentDirectory: "/private") == expected) + #expect(await authorizer.auditLog().count == 1) + let rules = RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.glob], pathPrefix: "/private", effect: .deny(message: nil)), + .init(operations: [.glob], pathPrefix: "/", effect: .allow), + ]) + let ruled = AuthorizedFileSystem(base: base, authorizer: rules) + #expect(try await ruled.glob(pattern: "/public/*", currentDirectory: "/private") == expected) + try await base.createSymlink(path: "/public/alias", target: "../private") + await #expect(throws: WorkspaceError.self) { + _ = try await ruled.glob(pattern: "/public/alias/*", currentDirectory: "/") + } + } + + @Test + func authorizationPreservesSyntheticMountParents() async throws { + let leaf = InMemoryFileSystem() + try await leaf.createFile(path: "/file", data: Data("mounted".utf8)) + let mounted = MountedFileSystem( + base: InMemoryFileSystem(), + mounts: [.init(mountPoint: "/a/b", fileSystem: leaf)] + ) + let fs = AuthorizedFileSystem( + base: mounted, + authorizer: PermissionAuthorizer { + _ in .allow + }) + #expect(try await fs.listDirectory(path: "/a").map(\.name) == ["b"]) + #expect(await fs.exists(path: "/a")) + #expect(try await fs.stat(path: "/a").kind == .directory) + #expect(try await fs.readFile(path: "/a/b/file") == Data("mounted".utf8)) + } + + @Test + func absoluteSymlinkAcceptsRealpathRootSpelling() async throws { + let root = try TestSupport.temporaryDirectory("PhysicalRoot") + defer { + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + try await fs.createFile(path: "/file", data: Data("physical".utf8)) + let physical = try #require(realpath(root.path, nil)) + defer { + free(physical) + } + try await fs.createSymlink(path: "/link", target: String(cString: physical) + "/file") + #expect(try await fs.readFile(path: "/link") == Data("physical".utf8)) + #expect(try await fs.resolveRealPath(path: "/link") == "/file") + } + + // Root can list mode-000 directories and would conceal the traversal regression. + @Test(.enabled(if: geteuid() != 0, "Root bypasses permission checks")) + func globSkipsUnreadableSubdirectoriesButReportsUnreadableRoot() async throws { + let root = try TestSupport.temporaryDirectory("GlobPermissions") + let locked = root.appendingPathComponent("locked") + defer { + _ = chmod(root.path, 0o755) + _ = chmod(locked.path, 0o755) + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/locked", recursive: false) + try await fs.createFile(path: "/locked/hidden.txt", data: Data()) + try await fs.createFile(path: "/visible.txt", data: Data()) + #expect(chmod(locked.path, 0o000) == 0) + #expect(try await fs.glob(pattern: "/**/*.txt", currentDirectory: "/") == ["/visible.txt"]) + #expect(chmod(root.path, 0o000) == 0) + await #expect(throws: (any Error).self) { + _ = try await fs.glob(pattern: "/*", currentDirectory: "/") + } + } + + // Search-only ancestors are a distinct permission contract from directory listing. + @Test(.enabled(if: geteuid() != 0, "Root bypasses permission checks")) + func localOperationsTraverseSearchOnlyAncestors() async throws { + let root = try TestSupport.temporaryDirectory("SearchOnly") + let ancestor = root.appendingPathComponent("ancestor") + defer { + _ = chmod(ancestor.path, 0o755) + TestSupport.remove(root) + } + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/ancestor/writable", recursive: true) + try await fs.createFile(path: "/ancestor/writable/file", data: Data("inside".utf8)) + #expect(chmod(ancestor.path, 0o111) == 0) + #expect(try await fs.readFile(path: "/ancestor/writable/file") == Data("inside".utf8)) + try await fs.createFile(path: "/ancestor/writable/new", data: Data()) + try await fs.move(from: "/ancestor/writable/new", to: "/ancestor/writable/moved") + try await fs.remove(path: "/ancestor/writable/moved", recursive: false) + #expect(try await fs.listDirectory(path: "/ancestor/writable").map(\.name) == ["file"]) + await #expect(throws: (any Error).self) { + _ = try await fs.listDirectory(path: "/ancestor") + } + } + + @Test + func relativeParentSymlinkCannotEscapeRoot() async throws { + let container = try TestSupport.temporaryDirectory("RelativeEscape") + defer { + TestSupport.remove(container) + } + let fs = try LocalFileSystem(root: container.appendingPathComponent("root")) + let outside = try LocalFileSystem(root: container.appendingPathComponent("outside")) + try await outside.createFile(path: "/file", data: Data("secret".utf8)) + try await fs.createSymlink(path: "/esc", target: "../outside/file") + do { + _ = try await fs.readFile(path: "/esc") + Issue.record("expected relative symlink escape to be rejected") + } catch let error as WorkspaceError { + guard case .invalidPath = error else { + Issue.record("expected invalidPath, received \(error)") + return + } + } + } + + @Test + func concurrentMovesNeverReplaceTheWinningDestination() async throws { + let root = try TestSupport.temporaryDirectory("ExclusiveMove") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + let sources = (0..<24).map { WorkspacePath.root.appending("source-\($0)") } + for source in sources { + try await fs.createFile(path: source, data: Data(source.string.utf8)) + } + let winners = await withTaskGroup(of: WorkspacePath?.self) { group in + for source in sources { + group.addTask { + do { + try await fs.move(from: source, to: "/destination") + return source + } catch let error as NSError { + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(EEXIST)) + return nil + } + } + } + var winners: [WorkspacePath] = [] + for await winner in group { + if let winner { + winners.append(winner) + } + } + return winners + } + #expect(winners.count == 1) + let winner = try #require(winners.first) + #expect(try await fs.readFile(path: "/destination") == Data(winner.string.utf8)) + for source in sources { + #expect(await fs.exists(path: source) == (source != winner)) + } + } + + @Test + func unsupportedExclusiveRenameFallsBackWithoutReplacingExistingEntries() async throws { + #if canImport(Darwin) + let unsupportedErrors = [ENOTSUP, ENOSYS] + #else + let unsupportedErrors = [EINVAL, ENOSYS] + #endif + for unsupportedError in unsupportedErrors { + let root = try TestSupport.temporaryDirectory("RenameFallback") + defer { TestSupport.remove(root) } + let fs = LocalFileSystem(exclusiveRename: { _, _, _, _ in unsupportedError }) + try fs.applyConfiguration(rootDirectory: root) + try await fs.createFile(path: "/source", data: Data("source".utf8)) + try await fs.move(from: "/source", to: "/destination") + #expect(try await fs.readFile(path: "/destination") == Data("source".utf8)) + #expect(!(await fs.exists(path: "/source"))) + + try await fs.createFile(path: "/source", data: Data("keep".utf8)) + try await fs.createSymlink(path: "/dangling", target: "missing") + for destination: WorkspacePath in ["/destination", "/dangling"] { + do { + try await fs.move(from: "/source", to: destination) + Issue.record("expected EEXIST for an occupied destination") + } catch let error as NSError { + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(EEXIST)) + } + } + #expect(try await fs.readFile(path: "/source") == Data("keep".utf8)) + #expect(try await fs.readFile(path: "/destination") == Data("source".utf8)) + #expect(try await fs.readSymlink(path: "/dangling") == "missing") + + try await fs.createDirectory(path: "/directory/child", recursive: true) + try await fs.move(from: "/directory", to: "/renamed") + #expect(try await fs.stat(path: "/renamed/child").kind == .directory) + #expect(!(await fs.exists(path: "/directory"))) + do { + try await fs.move(from: "/renamed", to: "/renamed/child/invalid") + Issue.record("expected EINVAL for a directory moved into itself") + } catch let error as NSError { + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(EINVAL)) + } + } + } + + @Test + func exclusiveRenamePermissionFailuresDoNotFallBack() async throws { + let root = try TestSupport.temporaryDirectory("RenameDenied") + defer { TestSupport.remove(root) } + let fs = LocalFileSystem(exclusiveRename: { _, _, _, _ in EACCES }) + try fs.applyConfiguration(rootDirectory: root) + try await fs.createFile(path: "/source", data: Data("keep".utf8)) + do { + try await fs.move(from: "/source", to: "/destination") + Issue.record("expected the exclusive rename error to propagate") + } catch let error as NSError { + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(EACCES)) + } + #expect(try await fs.readFile(path: "/source") == Data("keep".utf8)) + #expect(!(await fs.exists(path: "/destination"))) + } +} From 715d0ba2992a8da4115540bc242677928d92316b Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:09:22 -0700 Subject: [PATCH 05/34] Keep LocalFileSystem.remove a no-op for paths beneath a regular file --- Sources/Workspace/FS/LocalFileSystem.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 5a90212..7e92d92 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -489,7 +489,10 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { } do { try removeEntry(resolve(path, followFinal: false), recursive: recursive) - } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { + } catch let error as NSError + where error.domain == NSPOSIXErrorDomain && (error.code == Int(ENOENT) || error.code == Int(ENOTDIR)) + { + // A path beneath a regular file can't exist, so removing it is the same no-op as a missing path. return } } From e1073ddc0ddd86ad3cd18af7bd2adf3efa86151a Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:13:45 -0700 Subject: [PATCH 06/34] Fix checkpoint-store deadlock, torn-log recovery and cross-instance history FileCheckpointStore serializes in-process callers through an async gate before touching the OS flock, takes uncontended locks inline, and waits for contended ones off the cooperative pool, so concurrent checkpoint capture and queries on one store no longer deadlock. mutations.jsonl is split on raw LF bytes and decoded per record, so an interrupted append that ends mid-UTF-8 character no longer empties history; only an incomplete final record is tolerated. Checkpoint reads validate negative lengths and forward ranges to the store. Checkpoint mutation cursors come from the store's sequence high-water mark, read under its lock before capture starts. history() refreshes persisted records incrementally by sequence, crosses permanent gaps, and does a full reload when another instance retracts or prunes records (tracked by a retraction epoch stored with the high-water mark; legacy sidecars still decode). Checkpoint event delivery is tracked separately from the query cache. history() now reads the store and waits for in-flight mutating operations, so it reflects other instances and throws if the stored log is unreadable instead of returning a stale cache. Size+mtime blob reuse follows git's racy-entry rule, and Workspace(checkpointCapture: .verifyContents) opts into always rehashing. --- README.md | 15 +- Sources/Workspace/Checkpoint.swift | 13 +- Sources/Workspace/CheckpointStore.swift | 448 +++++++++++++----- Sources/Workspace/Workspace+Checkpoints.swift | 4 +- Sources/Workspace/Workspace+Internals.swift | 33 +- Sources/Workspace/Workspace.swift | 26 +- Sources/Workspace/WorkspaceEvent.swift | 3 + Sources/Workspace/WorkspaceHistory.swift | 31 ++ Sources/Workspace/WorkspaceRevision.swift | 27 +- .../WorkspaceTests/CheckpointStoreTests.swift | 73 ++- Tests/WorkspaceTests/CompactionTests.swift | 3 +- Tests/WorkspaceTests/EditAtomicityTests.swift | 5 + Tests/WorkspaceTests/PersistenceTests.swift | 343 +++++++++++++- 13 files changed, 852 insertions(+), 172 deletions(-) diff --git a/README.md b/README.md index f6bdafd..8500cfd 100644 --- a/README.md +++ b/README.md @@ -170,12 +170,19 @@ Compaction rebases retained checkpoints to their nearest retained ancestor and r Directory-backed stores serialize checkpoint commits, revision reads, and compaction with a persistent lifecycle lock. Mutation history is independent and is not pruned by checkpoint retention. -Store format version 1 is retained so existing stores remain readable. Once a store contains `mutations.sequence.json`, do not use older library versions that ignore this file to write to that store, including through concurrent instances or after a downgrade: those writers can reuse retracted mutation sequence numbers. All writers must preserve the sequence high-water mark. +Store format version 1 is retained so existing stores remain readable. Once a store contains `mutations.sequence.json`, do not use older library versions that ignore this file to write to that store, including through concurrent instances or after a downgrade: those writers can reuse retracted mutation sequence numbers. All writers must preserve the sequence high-water mark and retraction epoch. The sidecar now stores both fields; legacy bare-integer sidecars remain readable. Retractions and failed append reservations can leave sequence gaps. History refreshes invalidate cached records when another instance retracts or prunes mutations. Directory-backed checkpoint capture also reuses existing CAS blobs when a file's size and available -modification timestamp are unchanged. It still walks directory metadata for correctness, but only -reads and hashes file contents that appear changed. Filesystems that do not provide modification -timestamps safely fall back to reading those files. +modification timestamp are unchanged and the timestamp is more than two seconds older than the +previous capture's start. Entries within that timestamp-granularity window are rehashed. It still +walks directory metadata for correctness. Filesystems without modification timestamps, and older +manifests without a capture timestamp, safely fall back to reading those files. + +For tools that preserve older timestamps while changing content, pass +`checkpointCapture: .verifyContents` to `Workspace` to read and hash every file at each checkpoint. +The default `.trustMetadata` policy compares the local wall clock with filesystem modification +timestamps. Clock skew can make a recent change appear safely old; use `.verifyContents` for +network filesystems whose clock may differ. This policy affects only `.directory` persistence. ## Transactions diff --git a/Sources/Workspace/Checkpoint.swift b/Sources/Workspace/Checkpoint.swift index ed16e2f..76fcc96 100644 --- a/Sources/Workspace/Checkpoint.swift +++ b/Sources/Workspace/Checkpoint.swift @@ -81,6 +81,7 @@ public struct Checkpoint: Sendable, Codable, Equatable { static func make( snapshot: Snapshot, draft: CheckpointDraft, + mutationCursor: Int, parent: Checkpoint?, parentSnapshot: Snapshot? ) -> Checkpoint { @@ -88,7 +89,7 @@ public struct Checkpoint: Sendable, Codable, Equatable { snapshot: snapshot, parent: parent, parentSnapshot: parentSnapshot, - mutationCursor: draft.mutationCursor + mutationCursor: mutationCursor ) return Checkpoint( workspaceId: draft.workspaceID, @@ -97,7 +98,7 @@ public struct Checkpoint: Sendable, Codable, Equatable { origin: draft.origin, firstMutationSequence: metadata.firstMutationSequence, lastMutationSequence: metadata.lastMutationSequence, - mutationCursor: draft.mutationCursor, + mutationCursor: mutationCursor, snapshotId: snapshot.id, summary: metadata.summary ) @@ -106,6 +107,7 @@ public struct Checkpoint: Sendable, Codable, Equatable { static func make( snapshotID: UUID, draft: CheckpointDraft, + mutationCursor: Int, parent: Checkpoint?, summary: ChangeSet.Summary ) -> Checkpoint { @@ -115,9 +117,9 @@ public struct Checkpoint: Sendable, Codable, Equatable { label: draft.label, parentCheckpointId: parent?.id, origin: draft.origin, - firstMutationSequence: draft.mutationCursor > previousCursor ? previousCursor + 1 : nil, - lastMutationSequence: draft.mutationCursor > previousCursor ? draft.mutationCursor : nil, - mutationCursor: draft.mutationCursor, + firstMutationSequence: mutationCursor > previousCursor ? previousCursor + 1 : nil, + lastMutationSequence: mutationCursor > previousCursor ? mutationCursor : nil, + mutationCursor: mutationCursor, snapshotId: snapshotID, summary: summary ) @@ -162,5 +164,4 @@ struct CheckpointDraft: Sendable { var label: String? var preferredParentID: UUID? var origin: Checkpoint.Origin - var mutationCursor: Int } diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 349e005..351f254 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -29,7 +29,7 @@ protocol CheckpointStore: AnyObject, Sendable { ) async throws -> Data? @discardableResult func appendMutation(_ mutation: Mutation) async throws -> Mutation - func listMutations(workspaceId: UUID) async throws -> [Mutation] + func listMutations(workspaceId: UUID, afterSequence: Int) async throws -> MutationPage /// Retracts one failed operation's record without lowering the workspace's /// sequence high-water mark. Later appends must never reuse its sequence. func removeMutation(sequence: Int, workspaceId: UUID) async throws @@ -44,12 +44,25 @@ protocol CheckpointStore: AnyObject, Sendable { ) async throws -> StoreCompactionResult } +/// Records and invalidation metadata from the same locked read. +struct MutationPage: Sendable { + var records: [Mutation] + var retractionEpoch: Int +} + +extension CheckpointStore { + func listMutations(workspaceId: UUID) async throws -> [Mutation] { + try await listMutations(workspaceId: workspaceId, afterSequence: 0).records + } +} + /// An in-memory checkpoint store for tests and ephemeral workspaces. actor InMemoryCheckpointStore: CheckpointStore { private var checkpointsByWorkspace: [UUID: [Checkpoint]] = [:] private var snapshotsByWorkspace: [UUID: [UUID: Snapshot]] = [:] private var mutationsByWorkspace: [UUID: [Mutation]] = [:] private var mutationHighWaterMarks: [UUID: Int] = [:] + private var mutationRetractionEpochs: [UUID: Int] = [:] init() {} @@ -61,10 +74,20 @@ actor InMemoryCheckpointStore: CheckpointStore { } func captureRevision(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint { - try await saveRevision(Snapshot.capture(from: filesystem), draft: draft) + let mutationCursor = persistedMutationCursor(workspaceId: draft.workspaceID) + let snapshot = try await Snapshot.capture(from: filesystem) + return try persistRevision(snapshot, draft: draft, mutationCursor: mutationCursor) } func saveRevision(_ snapshot: Snapshot, draft: CheckpointDraft) async throws -> Checkpoint { + try persistRevision(snapshot, draft: draft, mutationCursor: persistedMutationCursor(workspaceId: draft.workspaceID)) + } + + private func persistedMutationCursor(workspaceId: UUID) -> Int { + mutationHighWaterMarks[workspaceId] ?? 0 + } + + private func persistRevision(_ snapshot: Snapshot, draft: CheckpointDraft, mutationCursor: Int) throws -> Checkpoint { var list = checkpointsByWorkspace[draft.workspaceID] ?? [] let parent = resolvedParent(for: draft, checkpoints: list) let parentSnapshot = parent.flatMap { snapshotsByWorkspace[draft.workspaceID]?[$0.snapshotId] } @@ -74,6 +97,7 @@ actor InMemoryCheckpointStore: CheckpointStore { let checkpoint = Checkpoint.make( snapshot: snapshot, draft: draft, + mutationCursor: mutationCursor, parent: parent, parentSnapshot: parentSnapshot ) @@ -111,14 +135,17 @@ actor InMemoryCheckpointStore: CheckpointStore { offset: UInt64, length: Int? ) async throws -> Data? { + if let length, length < 0 { + throw WorkspaceError.unsupported("read length must not be negative") + } guard let snapshot = snapshotsByWorkspace[workspaceId]?[id], let entry = Workspace.snapshotEntry(at: path, in: snapshot.entry), case let .file(file) = entry else { return nil } guard offset < UInt64(file.data.count) else { return Data() } let start = file.data.index(file.data.startIndex, offsetBy: Int(offset)) - let end = length.map { file.data.index(start, offsetBy: $0, limitedBy: file.data.endIndex) ?? file.data.endIndex } - ?? file.data.endIndex + let count = min(length ?? file.data.count, file.data.count - Int(offset)) + let end = file.data.index(start, offsetBy: count) return Data(file.data[start.. [Mutation] { - (mutationsByWorkspace[workspaceId] ?? []).sorted { $0.sequence < $1.sequence } + func listMutations(workspaceId: UUID, afterSequence: Int) async throws -> MutationPage { + MutationPage( + records: (mutationsByWorkspace[workspaceId] ?? []).filter { $0.sequence > afterSequence }.sorted { $0.sequence < $1.sequence }, + retractionEpoch: mutationRetractionEpochs[workspaceId] ?? 0 + ) } func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws { @@ -151,6 +182,7 @@ actor InMemoryCheckpointStore: CheckpointStore { if kept.isEmpty { kept = [last] } + if kept.count != records.count { mutationRetractionEpochs[workspaceId, default: 0] += 1 } mutationsByWorkspace[workspaceId] = kept } @@ -242,6 +274,8 @@ actor InMemoryCheckpointStore: CheckpointStore { /// instances in the same process and across cooperating processes do not lose appends. A separate /// lifecycle lock serializes checkpoint commits, revision reads, and compaction; rebased metadata /// is written before unreachable artifacts are removed, keeping interrupted compaction recoverable. +/// In-process async gates serialize reentrant callers before OS locking; contended OS locks wait +/// off the cooperative executor so a suspended capture can resume and release its lock. /// Coordinating writers on network filesystems that do not honor `flock` may still require /// application-level serialization. actor FileCheckpointStore: CheckpointStore { @@ -251,11 +285,21 @@ actor FileCheckpointStore: CheckpointStore { private let decoder: JSONDecoder private let compactEncoder: JSONEncoder + private let checkpointCapture: Workspace.CheckpointCapturePolicy + // Actor reentrancy must suspend contenders before they can conflict with this store's own fd. + private var heldLocks: Set = [] + private var lockWaiters: [String: [CheckedContinuation]] = [:] + private var listCheckpointsCache: [UUID: (cacheKey: String, checkpoints: [Checkpoint])] = [:] - init(rootDirectory: URL, fileManager: FileManager = .default) { + init( + rootDirectory: URL, + fileManager: FileManager = .default, + checkpointCapture: Workspace.CheckpointCapturePolicy = .trustMetadata + ) { self.rootDirectory = rootDirectory.standardizedFileURL self.fileManager = fileManager + self.checkpointCapture = checkpointCapture let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys] @@ -265,17 +309,19 @@ actor FileCheckpointStore: CheckpointStore { } func prepare(workspaceId: UUID) async throws { - try ensureWorkspaceDirectories(for: workspaceId) + try await ensureWorkspaceDirectories(for: workspaceId) } func changes(workspaceId: UUID) async throws -> AsyncStream { - try ensureWorkspaceDirectories(for: workspaceId) + try await ensureWorkspaceDirectories(for: workspaceId) return try DirectoryChangeStream.observe(checkpointsDirectoryURL(workspaceId: workspaceId)) } func saveRevision(_ snapshot: Snapshot, draft: CheckpointDraft) async throws -> Checkpoint { - try ensureWorkspaceDirectories(for: draft.workspaceID) - return try Self.withExclusiveLock(at: lifecycleLockURL(workspaceId: draft.workspaceID)) { + try await ensureWorkspaceDirectories(for: draft.workspaceID) + return try await withExclusiveLock(at: lifecycleLockURL(workspaceId: draft.workspaceID)) { + // The snapshot was captured by the caller before acquiring the lifecycle lock. + let mutationCursor = try await persistedMutationCursor(workspaceId: draft.workspaceID) let checkpoints = try loadAllCheckpointsUnlocked(workspaceId: draft.workspaceID) let parent = resolvedParent(for: draft, checkpoints: checkpoints) let parentSnapshot = try parent.map { @@ -287,6 +333,7 @@ actor FileCheckpointStore: CheckpointStore { let checkpoint = Checkpoint.make( snapshot: snapshot, draft: draft, + mutationCursor: mutationCursor, parent: parent, parentSnapshot: parentSnapshot ) @@ -298,9 +345,9 @@ actor FileCheckpointStore: CheckpointStore { } func loadCheckpoint(id: UUID, workspaceId: UUID) async throws -> Checkpoint? { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } - return try Self.withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { let url = checkpointURL(id: id, workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return nil } let checkpoint = try read(Checkpoint.self, from: url) @@ -312,13 +359,13 @@ actor FileCheckpointStore: CheckpointStore { } func listCheckpoints(workspaceId: UUID) async throws -> [Checkpoint] { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) let directoryURL = checkpointsDirectoryURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: directoryURL.path) else { listCheckpointsCache[workspaceId] = nil return [] } - return try Self.withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { if let key = try checkpointsDirectoryCacheKey(at: directoryURL), let entry = listCheckpointsCache[workspaceId], entry.cacheKey == key { return entry.checkpoints @@ -355,11 +402,14 @@ actor FileCheckpointStore: CheckpointStore { var id: UUID var rootPath: WorkspacePath var root: Node + // Older manifests and materialized snapshots have no trustworthy capture timestamp. + var captureStartedAt: Date? = nil } func captureRevision(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint { - try ensureWorkspaceDirectories(for: draft.workspaceID) - return try await withExclusiveLifecycleLock(at: lifecycleLockURL(workspaceId: draft.workspaceID)) { + try await ensureWorkspaceDirectories(for: draft.workspaceID) + return try await withExclusiveLock(at: lifecycleLockURL(workspaceId: draft.workspaceID)) { + let mutationCursor = try await self.persistedMutationCursor(workspaceId: draft.workspaceID) let checkpoints = try self.loadAllCheckpointsUnlocked(workspaceId: draft.workspaceID) let parent = self.resolvedParent(for: draft, checkpoints: checkpoints) let parentManifest: SnapshotManifest? @@ -374,14 +424,18 @@ actor FileCheckpointStore: CheckpointStore { } else { parentManifest = nil } + let captureStartedAt = Date() let snapshotID = UUID() let root = try await self.captureManifestNode( from: filesystem, at: .root, reusing: parentManifest?.root, + previousCaptureStartedAt: parentManifest?.captureStartedAt, workspaceId: draft.workspaceID ) - let manifest = SnapshotManifest(version: 2, id: snapshotID, rootPath: .root, root: root) + let manifest = SnapshotManifest( + version: 2, id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt + ) let beforeIndex = try parentManifest.map { RevisionIndex( id: $0.id, @@ -410,6 +464,7 @@ actor FileCheckpointStore: CheckpointStore { let checkpoint = Checkpoint.make( snapshotID: snapshotID, draft: draft, + mutationCursor: mutationCursor, parent: parent, summary: changes.summary ) @@ -448,6 +503,7 @@ actor FileCheckpointStore: CheckpointStore { from filesystem: any FileSystem, at path: WorkspacePath, reusing previous: SnapshotManifest.Node?, + previousCaptureStartedAt: Date?, workspaceId: UUID ) async throws -> SnapshotManifest.Node { guard await filesystem.exists(path: path) else { @@ -456,7 +512,13 @@ actor FileCheckpointStore: CheckpointStore { let info = try await filesystem.stat(path: path) switch info.kind { case .file: - if previous?.kind == .file, + // Conservatively cover second-resolution and two-second-resolution timestamps. + // Equality alone is unsafe when an entry was captured in its timestamp's tick. + if checkpointCapture == .trustMetadata, + let previousCaptureStartedAt, + let modificationDate = info.modificationDate, + modificationDate.addingTimeInterval(2) < previousCaptureStartedAt, + previous?.kind == .file, previous?.contentSize == info.size, previous?.modificationDate != nil, previous?.modificationDate == info.modificationDate, @@ -500,6 +562,7 @@ actor FileCheckpointStore: CheckpointStore { from: filesystem, at: path.appending(entry.name), reusing: previousChildren[entry.name], + previousCaptureStartedAt: previousCaptureStartedAt, workspaceId: workspaceId ) ) @@ -549,17 +612,17 @@ actor FileCheckpointStore: CheckpointStore { } func loadSnapshot(id: UUID, workspaceId: UUID) async throws -> Snapshot? { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } - return try Self.withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { try loadSnapshotUnlocked(id: id, workspaceId: workspaceId) } } func loadRevisionIndex(id: UUID, workspaceId: UUID) async throws -> RevisionIndex? { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } - return try Self.withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { let url = snapshotURL(id: id, workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return nil } let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) @@ -581,12 +644,12 @@ actor FileCheckpointStore: CheckpointStore { offset: UInt64, length: Int? ) async throws -> Data? { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) if let length, length < 0 { throw WorkspaceError.unsupported("read length must not be negative") } guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } - return try Self.withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { let url = snapshotURL(id: id, workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return nil } let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) @@ -606,7 +669,8 @@ actor FileCheckpointStore: CheckpointStore { let size = try handle.seekToEnd() guard offset < size else { return Data() } try handle.seek(toOffset: offset) - return try handle.read(upToCount: length ?? Int(size - offset)) ?? Data() + let count = min(UInt64(length ?? Int.max), size - offset) + return try handle.read(upToCount: Int(count)) ?? Data() } } @@ -734,26 +798,19 @@ actor FileCheckpointStore: CheckpointStore { @discardableResult func appendMutation(_ mutation: Mutation) async throws -> Mutation { - try ensureWorkspaceDirectories(for: mutation.workspaceID) + try await ensureWorkspaceDirectories(for: mutation.workspaceID) let jsonl = mutationsJsonlURL(workspaceId: mutation.workspaceID) let lockURL = mutationsLockURL(workspaceId: mutation.workspaceID) - return try Self.withExclusiveLock(at: lockURL) { - let lastSequence: Int - if fileManager.fileExists(atPath: jsonl.path) { - try repairTornTail(at: jsonl) - lastSequence = try lastPersistedSequence(at: jsonl) - } else { - lastSequence = 0 - } - var record = mutation - let highWaterMark = try max(lastSequence, mutationHighWaterMark(workspaceId: mutation.workspaceID)) - guard highWaterMark < Int.max else { + return try await withExclusiveLock(at: lockURL) { + var state = try mutationSequenceState(workspaceId: mutation.workspaceID) + guard state.highWaterMark < Int.max else { throw WorkspaceError.storageCorrupted("mutation sequence exhausted") } - record.sequence = highWaterMark + 1 - // Reserve before publishing the record. A failed append may leave a - // gap, but neither a failure nor a retraction can reuse an observed ID. - try write(record.sequence, to: mutationSequenceURL(workspaceId: mutation.workspaceID)) + var record = mutation + state.highWaterMark += 1 + record.sequence = state.highWaterMark + // Reserve before publishing. Failed appends and retractions cannot reuse an ID. + try write(state, to: mutationSequenceURL(workspaceId: mutation.workspaceID)) if fileManager.fileExists(atPath: jsonl.path) { try appendJSONLLine(encode: record, to: jsonl) @@ -765,39 +822,43 @@ actor FileCheckpointStore: CheckpointStore { } func removeMutation(sequence: Int, workspaceId: UUID) async throws { - try Self.withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + try await ensureWorkspaceDirectories(for: workspaceId) + try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { let url = mutationsJsonlURL(workspaceId: workspaceId) - let records = try loadMutationsFromJSONL(at: url) - let highWaterMark = try max( - records.map(\.sequence).max() ?? 0, - mutationHighWaterMark(workspaceId: workspaceId) - ) - try write(highWaterMark, to: mutationSequenceURL(workspaceId: workspaceId)) + var state = try mutationSequenceState(workspaceId: workspaceId) + let records = fileManager.fileExists(atPath: url.path) ? try loadMutationsFromJSONL(at: url) : [] + try state.bumpRetractionEpoch() + // Publish invalidation before rewriting. A failed rewrite only causes a harmless reload. + try write(state, to: mutationSequenceURL(workspaceId: workspaceId)) try writeAllMutationsAsJSONL(records.filter { $0.sequence != sequence }, to: url) } } - func listMutations(workspaceId: UUID) async throws -> [Mutation] { - try validateWorkspaceFormatIfPresent(workspaceId) - let jsonl = mutationsJsonlURL(workspaceId: workspaceId) - guard fileManager.fileExists(atPath: jsonl.path) else { - return [] + func listMutations(workspaceId: UUID, afterSequence: Int) async throws -> MutationPage { + try await validateWorkspaceFormatIfPresent(workspaceId) + guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { + return MutationPage(records: [], retractionEpoch: 0) } - let lockURL = mutationsLockURL(workspaceId: workspaceId) - return try Self.withExclusiveLock(at: lockURL) { - try loadMutationsFromJSONL(at: jsonl) - .sorted { $0.sequence < $1.sequence } + return try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + let jsonl = mutationsJsonlURL(workspaceId: workspaceId) + let state = try readMutationSequenceState(workspaceId: workspaceId) + let records = fileManager.fileExists(atPath: jsonl.path) + ? try loadMutationsFromJSONL(at: jsonl, afterSequence: afterSequence).sorted { $0.sequence < $1.sequence } + : [] + return MutationPage(records: records, retractionEpoch: state.retractionEpoch) } } func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) let jsonl = mutationsJsonlURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: jsonl.path) else { return } let lockURL = mutationsLockURL(workspaceId: workspaceId) - try Self.withExclusiveLock(at: lockURL) { + try await withExclusiveLock(at: lockURL) { + var state = try mutationSequenceState(workspaceId: workspaceId) + guard fileManager.fileExists(atPath: jsonl.path) else { return } let records = try loadMutationsFromJSONL(at: jsonl) .sorted { $0.sequence < $1.sequence } guard let last = records.last else { @@ -810,16 +871,18 @@ actor FileCheckpointStore: CheckpointStore { guard kept.count != records.count else { return } + try state.bumpRetractionEpoch() + try write(state, to: mutationSequenceURL(workspaceId: workspaceId)) try writeAllMutationsAsJSONL(kept, to: jsonl) } } func storageStatistics(workspaceId: UUID) async throws -> Workspace.StorageStatistics { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return .empty } - return try Self.withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { try physicalStatisticsUnlocked(workspaceId: workspaceId) } } @@ -829,7 +892,7 @@ actor FileCheckpointStore: CheckpointStore { retaining retention: Workspace.Retention, dryRun: Bool ) async throws -> StoreCompactionResult { - try validateWorkspaceFormatIfPresent(workspaceId) + try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { let plan = try CheckpointRetentionPlanner.plan(checkpoints: [], retaining: retention) return StoreCompactionResult( @@ -843,7 +906,7 @@ actor FileCheckpointStore: CheckpointStore { checkpoints: [] ) } - return try Self.withExclusiveLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + return try await withExclusiveLock(at: lifecycleLockURL(workspaceId: workspaceId)) { try compactUnlocked(workspaceId: workspaceId, retaining: retention, dryRun: dryRun) } } @@ -1041,23 +1104,25 @@ actor FileCheckpointStore: CheckpointStore { if fileManager.fileExists(atPath: url.path) { try fileManager.removeItem(at: url) } } - private func loadMutationsFromJSONL(at url: URL) throws -> [Mutation] { + private func loadMutationsFromJSONL(at url: URL, afterSequence: Int = 0) throws -> [Mutation] { + if afterSequence > 0 { + return try loadMutationSuffix(at: url, afterSequence: afterSequence) + } let data = try Data(contentsOf: url) if data.isEmpty { return [] } let endsWithNewline = data.last == UInt8(ascii: "\n") - let text = String(data: data, encoding: .utf8) ?? "" - let lines = text.split(whereSeparator: \.isNewline) + let lines = data.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false) var out: [Mutation] = [] out.reserveCapacity(lines.count) for (index, line) in lines.enumerated() { if line.isEmpty { continue } do { - out.append(try decoder.decode(Mutation.self, from: Data(String(line).utf8))) + out.append(try decoder.decode(Mutation.self, from: Data(line))) } catch { // A crashed append leaves a strict prefix of `\n`: an undecodable final // line with no trailing newline. Tolerate exactly that torn tail; anything else // is real corruption and must surface. - if index == lines.count - 1, !endsWithNewline { + if index == lines.count - 1, !endsWithNewline, Self.isIncompleteMutationRecord(Data(line)) { continue } throw error @@ -1066,10 +1131,100 @@ actor FileCheckpointStore: CheckpointStore { return out } + /// A closed object with invalid UTF-8 or an invalid Mutation schema is corruption, even + /// without its final LF. Track string escapes so a brace inside a truncated string does + /// not look like the end of the record. + private static func isIncompleteMutationRecord(_ data: Data) -> Bool { + let whitespace: Set = [0x20, 0x09, 0x0A, 0x0D] + guard data.first(where: { !whitespace.contains($0) }) == UInt8(ascii: "{") else { return false } + var delimiters: [UInt8] = [] + var inString = false + var escaped = false + var closed = false + for byte in data { + if closed { + if whitespace.contains(byte) { continue } + return false + } + if inString { + if escaped { escaped = false } + else if byte == UInt8(ascii: "\\") { escaped = true } + else if byte == UInt8(ascii: "\"") { inString = false } + continue + } + switch byte { + case UInt8(ascii: "\""): + inString = true + case UInt8(ascii: "{"), UInt8(ascii: "["): + delimiters.append(byte) + case UInt8(ascii: "}"), UInt8(ascii: "]"): + let opening = byte == UInt8(ascii: "}") ? UInt8(ascii: "{") : UInt8(ascii: "[") + guard delimiters.popLast() == opening else { return false } + closed = delimiters.isEmpty + default: + break + } + } + return !delimiters.isEmpty + } + + /// Scan back only as far as the requested sequence. Appends and pruning preserve sequence + /// order, so repeated history queries decode new records rather than the entire log. + private func loadMutationSuffix(at url: URL, afterSequence: Int) throws -> [Mutation] { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var cursor = try handle.seekToEnd() + guard cursor > 0 else { return [] } + try handle.seek(toOffset: cursor - 1) + let endsWithNewline = try handle.read(upToCount: 1) == Data([UInt8(ascii: "\n")]) + var pending = Data() + var isFinalRecord = true + var records: [Mutation] = [] + + func consume(_ line: Data) throws -> Bool { + defer { isFinalRecord = false } + guard !line.isEmpty else { return true } + let record: Mutation + do { + record = try decoder.decode(Mutation.self, from: line) + } catch { + if isFinalRecord, !endsWithNewline, Self.isIncompleteMutationRecord(line) { return true } + throw error + } + guard record.sequence > afterSequence else { return false } + records.append(record) + return true + } + + while cursor > 0 { + let start = cursor > 65536 ? cursor - 65536 : 0 + try handle.seek(toOffset: start) + var chunk = try handle.read(upToCount: Int(cursor - start)) ?? Data() + chunk.append(pending) + let lines = chunk.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false) + for line in lines.dropFirst().reversed() { + if try !consume(Data(line)) { return records.reversed() } + } + pending = Data(lines.first ?? Data.SubSequence()) + cursor = start + } + _ = try consume(pending) + return records.reversed() + } + + /// Call while holding the lifecycle lock. The cursor is authoritative even when the + /// Workspace that requested the checkpoint has not refreshed its mutation cache. + private func persistedMutationCursor(workspaceId: UUID) async throws -> Int { + try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + try mutationSequenceState(workspaceId: workspaceId).highWaterMark + } + } + /// Truncates a partial trailing line left behind by a crashed append so subsequent appends /// do not glue new records onto the torn fragment. A torn append never contains a newline /// (it is a strict prefix of `\n`), so everything after the final newline — or the - /// whole file when none exists — is the torn region. Must be called under the mutations lock. + /// whole file when none exists — is the final record. A complete record missing only its LF + /// is preserved by adding that LF. Must be called under the mutations lock. private func repairTornTail(at url: URL) throws { let handle = try FileHandle(forUpdating: url) defer { try? handle.close() } @@ -1079,7 +1234,18 @@ actor FileCheckpointStore: CheckpointStore { let lastByte = try handle.read(upToCount: 1) guard lastByte != Data("\n".utf8) else { return } let newlineOffset = try Self.lastNewlineOffset(in: handle, before: size - 1) - try handle.truncate(atOffset: newlineOffset.map { $0 + 1 } ?? 0) + let start = newlineOffset.map { $0 + 1 } ?? 0 + try handle.seek(toOffset: start) + let tail = try handle.read(upToCount: Int(size - start)) ?? Data() + do { + _ = try decoder.decode(Mutation.self, from: tail) + } catch { + guard Self.isIncompleteMutationRecord(tail) else { throw error } + try handle.truncate(atOffset: start) + return + } + try handle.seekToEnd() + try handle.write(contentsOf: Data("\n".utf8)) } /// Returns the sequence of the final record without loading the whole log. Appends are @@ -1166,9 +1332,9 @@ actor FileCheckpointStore: CheckpointStore { return "\(mtime?.timeIntervalSince1970 ?? 0)-\(count)" } - private func ensureWorkspaceDirectories(for workspaceId: UUID) throws { + private func ensureWorkspaceDirectories(for workspaceId: UUID) async throws { try fileManager.createDirectory(at: rootDirectory, withIntermediateDirectories: true) - try Self.withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { + try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { try ensureWorkspaceDirectoriesLocked(for: workspaceId) } } @@ -1193,10 +1359,10 @@ actor FileCheckpointStore: CheckpointStore { try fileManager.createDirectory(at: blobsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) } - private func validateWorkspaceFormatIfPresent(_ workspaceId: UUID) throws { + private func validateWorkspaceFormatIfPresent(_ workspaceId: UUID) async throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: workspaceURL.path) else { return } - try Self.withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { + try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { try validateWorkspaceFormatLocked(workspaceId) } } @@ -1245,13 +1411,45 @@ actor FileCheckpointStore: CheckpointStore { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("mutations.sequence.json") } - /// Must be read and updated under the mutations lock. Absent for older stores. - private func mutationHighWaterMark(workspaceId: UUID) throws -> Int { + private struct MutationSequenceState: Codable { + var highWaterMark: Int = 0 + var retractionEpoch: Int = 0 + + mutating func bumpRetractionEpoch() throws { + guard retractionEpoch < Int.max else { + throw WorkspaceError.storageCorrupted("mutation retraction epoch exhausted") + } + retractionEpoch += 1 + } + } + + /// Reads legacy bare-Int sidecars as epoch zero. Must hold the mutations lock. + private func readMutationSequenceState(workspaceId: UUID) throws -> MutationSequenceState { let url = mutationSequenceURL(workspaceId: workspaceId) - guard fileManager.fileExists(atPath: url.path) else { return 0 } - let value = try read(Int.self, from: url) - guard value >= 0 else { throw WorkspaceError.storageCorrupted("negative mutation sequence") } - return value + guard fileManager.fileExists(atPath: url.path) else { return MutationSequenceState() } + let data = try Data(contentsOf: url) + let state: MutationSequenceState + if let legacy = try? decoder.decode(Int.self, from: data) { + state = MutationSequenceState(highWaterMark: legacy) + } else { + state = try decoder.decode(MutationSequenceState.self, from: data) + } + guard state.highWaterMark >= 0, state.retractionEpoch >= 0 else { + throw WorkspaceError.storageCorrupted("negative mutation sequence or retraction epoch") + } + return state + } + + /// One authoritative cursor for reservation, retraction, pruning, and checkpoint capture. + /// Must hold the mutations lock; repair the tail before comparing the log with the sidecar. + private func mutationSequenceState(workspaceId: UUID) throws -> MutationSequenceState { + var state = try readMutationSequenceState(workspaceId: workspaceId) + let log = mutationsJsonlURL(workspaceId: workspaceId) + if fileManager.fileExists(atPath: log.path) { + try repairTornTail(at: log) + state.highWaterMark = max(state.highWaterMark, try lastPersistedSequence(at: log)) + } + return state } private func mutationsJsonlURL(workspaceId: UUID) -> URL { @@ -1295,60 +1493,70 @@ actor FileCheckpointStore: CheckpointStore { var version: Int } -#if canImport(Darwin) || canImport(Glibc) - private func withExclusiveLifecycleLock( - at url: URL, - _ body: () async throws -> R - ) async throws -> R { - let path = url.path - let fd = open(path, O_RDWR | O_CREAT, 0o644) - guard fd >= 0 else { throw LockFileError(message: "could not open lock file at \(path)") } - defer { close(fd) } - while flock(fd, LOCK_EX) != 0 { - if errno != EINTR { throw LockFileError(message: "could not acquire exclusive lock at \(path)") } + private func acquireGate(for path: String) async { + if heldLocks.insert(path).inserted { return } + await withCheckedContinuation { lockWaiters[path, default: []].append($0) } + } + + private func releaseGate(for path: String) { + guard var waiters = lockWaiters[path], !waiters.isEmpty else { + heldLocks.remove(path) + return } - defer { flock(fd, LOCK_UN) } - return try await body() + let next = waiters.removeFirst() + lockWaiters[path] = waiters.isEmpty ? nil : waiters + next.resume() } - private static func withExclusiveLock(at url: URL, _ body: () throws -> R) throws -> R { - try withLock(at: url, operation: LOCK_EX, body) + private enum LockMode: Sendable { case shared, exclusive } + + private func withExclusiveLock(at url: URL, _ body: () async throws -> R) async throws -> R { + try await withLock(at: url, mode: .exclusive, body) } - private static func withSharedLock(at url: URL, _ body: () throws -> R) throws -> R { - try withLock(at: url, operation: LOCK_SH, body) + private func withSharedLock(at url: URL, _ body: () async throws -> R) async throws -> R { + try await withLock(at: url, mode: .shared, body) } - private static func withLock(at url: URL, operation: Int32, _ body: () throws -> R) throws -> R { + private func withLock(at url: URL, mode: LockMode, _ body: () async throws -> R) async throws -> R { let path = url.path + await acquireGate(for: path) + defer { releaseGate(for: path) } +#if canImport(Darwin) || canImport(Glibc) let fd = open(path, O_RDWR | O_CREAT, 0o644) - guard fd >= 0 else { - throw LockFileError(message: "could not open lock file at \(path)") + guard fd >= 0 else { throw LockFileError(message: "could not open lock file at \(path)") } + defer { + _ = flock(fd, LOCK_UN) + close(fd) } - defer { close(fd) } - while flock(fd, operation) != 0 { - if errno != EINTR { - throw LockFileError(message: "could not acquire exclusive lock at \(path)") + let operation = mode == .exclusive ? LOCK_EX : LOCK_SH + while flock(fd, operation | LOCK_NB) != 0 { + let error = errno + if error == EINTR { continue } + guard error == EWOULDBLOCK else { + throw LockFileError(message: "could not acquire lock at \(path)") } + try await Self.waitForOSLock(fd, operation: operation, path: path) + break } - defer { _ = flock(fd, LOCK_UN) } - return try body() - } -#else - private func withExclusiveLifecycleLock( - at url: URL, - _ body: () async throws -> R - ) async throws -> R { - _ = url +#endif return try await body() } - private static func withExclusiveLock(at url: URL, _ body: () throws -> R) throws -> R { - try body() - } - - private static func withSharedLock(at url: URL, _ body: () throws -> R) throws -> R { - try body() +#if canImport(Darwin) || canImport(Glibc) + // flock may wait for another store or process. Never park a cooperative executor thread. + private nonisolated static func waitForOSLock(_ fd: Int32, operation: Int32, path: String) async throws { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + DispatchQueue.global().async { + while flock(fd, operation) != 0 { + if errno != EINTR { + continuation.resume(throwing: LockFileError(message: "could not acquire lock at \(path)")) + return + } + } + continuation.resume() + } + } } #endif } diff --git a/Sources/Workspace/Workspace+Checkpoints.swift b/Sources/Workspace/Workspace+Checkpoints.swift index 1fb34ed..fbbc794 100644 --- a/Sources/Workspace/Workspace+Checkpoints.swift +++ b/Sources/Workspace/Workspace+Checkpoints.swift @@ -14,10 +14,10 @@ extension Workspace { workspaceID: workspaceId, label: label, preferredParentID: headCheckpointId, - origin: .manual, - mutationCursor: latestMutationSequence() + origin: .manual ) ) + checkpoints.removeAll { $0.id == checkpoint.id } checkpoints.append(checkpoint) checkpoints.sort(by: checkpointSort) headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) diff --git a/Sources/Workspace/Workspace+Internals.swift b/Sources/Workspace/Workspace+Internals.swift index 2fab179..00ff4cf 100644 --- a/Sources/Workspace/Workspace+Internals.swift +++ b/Sources/Workspace/Workspace+Internals.swift @@ -59,7 +59,10 @@ extension Workspace { func loadStoreState() async throws { checkpoints = try await store.listCheckpoints(workspaceId: workspaceId) - mutations = try await store.listMutations(workspaceId: workspaceId) + let page = try await store.listMutations(workspaceId: workspaceId, afterSequence: 0) + mutations = page.records + historyRefreshSequence = mutations.last?.sequence ?? 0 + historyRetractionEpoch = page.retractionEpoch headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) didLoadStoreState = true } @@ -97,8 +100,7 @@ extension Workspace { workspaceID: workspaceId, label: label, preferredParentID: parentCheckpointId, - origin: origin, - mutationCursor: latestMutationSequence() + origin: origin ) ) checkpoints.removeAll { $0.id == checkpoint.id } @@ -115,8 +117,10 @@ extension Workspace { let persisted = try await store.appendMutation( Mutation(sequence: 0, workspaceID: workspaceId, operation: operation, changes: changes) ) - mutations.append(persisted) - mutations.sort { $0.sequence < $1.sequence } + cachePersistedMutations([persisted]) + if persisted.sequence == historyRefreshSequence + 1 { + historyRefreshSequence = persisted.sequence + } return persisted } @@ -144,10 +148,6 @@ extension Workspace { } } - func latestMutationSequence() -> Int { - mutations.last?.sequence ?? 0 - } - func encodedJSONString(for value: T, prettyPrinted: Bool) throws -> String { let encoder = JSONEncoder() if prettyPrinted { encoder.outputFormatting = [.prettyPrinted, .sortedKeys] } @@ -165,6 +165,9 @@ extension Workspace { let store = store let workspaceID = workspaceId guard let stream = try? await store.changes(workspaceId: workspaceID) else { return } + // Existing checkpoints are the subscription baseline. Query-cache refreshes after + // this point must not acknowledge event delivery. + deliveredCheckpointEventIDs.formUnion(checkpoints.map(\.id)) checkpointObservationTask = Task { [weak self] in for await _ in stream where !Task.isCancelled { try? await Task.sleep(for: .milliseconds(20)) @@ -181,16 +184,18 @@ extension Workspace { guard let stored = try? await store.listCheckpoints(workspaceId: workspaceId) else { return } let newCheckpoints = stored.filter { candidate in - !checkpoints.contains(where: { $0.id == candidate.id }) + !deliveredCheckpointEventIDs.contains(candidate.id) } checkpoints = stored.sorted(by: checkpointSort) headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) - if let refreshed = try? await store.listMutations(workspaceId: workspaceId) { - mutations = refreshed.sorted { $0.sequence < $1.sequence } - } + try? await refreshPersistedHistory() for checkpoint in newCheckpoints.sorted(by: checkpointSort) { - emitWorkspaceEvent(.checkpoint(checkpoint)) + // Re-check after suspension: a local commit or another refresh may have delivered it. + if !deliveredCheckpointEventIDs.contains(checkpoint.id) { + emitWorkspaceEvent(.checkpoint(checkpoint)) + } } + deliveredCheckpointEventIDs.formIntersection(checkpoints.map(\.id)) } func ensureFilesystemObservation() async { diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 872d749..0e6ffda 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -8,6 +8,15 @@ public actor Workspace { case directory(URL) } + /// Controls file-content verification during checkpoint capture with `.directory` persistence. + /// This policy has no effect on `.memory` persistence. + public enum CheckpointCapturePolicy: Sendable, Equatable { + /// Reuse matching size and mtime only when the mtime is safely before the prior capture. + case trustMetadata + /// Read and hash every file, including changes that preserve older timestamps. + case verifyContents + } + /// Controls only the detail stored in mutation history. Returned edit results, /// events, explicit previews, and diffs are independent of this policy and use /// `DiffOptions.default` unless the API accepts explicit diff options. @@ -45,18 +54,22 @@ public actor Workspace { var didPrepareStore = false var checkpoints: [Checkpoint] = [] var mutations: [Mutation] = [] + var historyRefreshSequence = 0 + var historyRetractionEpoch = 0 + var deliveredCheckpointEventIDs: Set = [] var headCheckpointId: UUID? var eventWatchers: [UUID: EventWatcher] = [:] var checkpointObservationTask: Task? var filesystemObservationTask: Task? var observedFilesystemSnapshot: Snapshot? - /// Creates a workspace over a filesystem. + /// Creates a workspace over a filesystem. `checkpointCapture` affects only `.directory` persistence. public init( workspaceID: UUID = UUID(), fileSystem: any FileSystem = InMemoryFileSystem(), persistence: Persistence = .memory, - recording: RecordingPolicy = .default + recording: RecordingPolicy = .default, + checkpointCapture: CheckpointCapturePolicy = .trustMetadata ) { self.workspaceId = workspaceID self.filesystem = fileSystem @@ -65,7 +78,7 @@ public actor Workspace { case .memory: InMemoryCheckpointStore() case let .directory(url): - FileCheckpointStore(rootDirectory: url) + FileCheckpointStore(rootDirectory: url, checkpointCapture: checkpointCapture) } } @@ -86,11 +99,14 @@ public actor Workspace { offset: UInt64, length: Int? = nil ) async throws -> Data { + if let length, length < 0 { + throw WorkspaceError.unsupported("read length must not be negative") + } switch revision { case .current: - try await filesystem.readFile(path: path, offset: offset, length: length) + return try await filesystem.readFile(path: path, offset: offset, length: length) case .checkpoint: - try await revisionData(at: path, revision: revision, offset: offset, length: length) + return try await revisionData(at: path, revision: revision, offset: offset, length: length) } } diff --git a/Sources/Workspace/WorkspaceEvent.swift b/Sources/Workspace/WorkspaceEvent.swift index 33e889f..f05af08 100644 --- a/Sources/Workspace/WorkspaceEvent.swift +++ b/Sources/Workspace/WorkspaceEvent.swift @@ -53,6 +53,9 @@ extension Workspace { } func emitWorkspaceEvent(_ event: WorkspaceEvent) { + if case let .checkpoint(checkpoint) = event { + guard deliveredCheckpointEventIDs.insert(checkpoint.id).inserted else { return } + } for watcher in eventWatchers.values where Self.matches(event, filter: watcher.filter) { watcher.continuation.yield(event) } diff --git a/Sources/Workspace/WorkspaceHistory.swift b/Sources/Workspace/WorkspaceHistory.swift index 4452728..109fda9 100644 --- a/Sources/Workspace/WorkspaceHistory.swift +++ b/Sources/Workspace/WorkspaceHistory.swift @@ -32,8 +32,39 @@ public struct Mutation: Sendable, Codable, Equatable { extension Workspace { /// Returns recorded mutations in ascending sequence order. + /// + /// History is read from the persistence store, so it includes records appended by other + /// workspace instances sharing the store and throws if the stored log can't be read. The call + /// waits for this workspace's in-flight mutating operations so it never reports a change whose + /// record hasn't been written yet, or one that is about to be retracted. public func history() async throws -> [Mutation] { + await acquireOperation() + defer { releaseOperation() } try await ensureLoaded() + try await refreshPersistedHistory() return mutations } + + func refreshPersistedHistory() async throws { + // A complete suffix read may cross permanent gaps from retractions or reservations. + // An epoch change invalidates the prefix too, including when the log is now empty. + var page = try await store.listMutations(workspaceId: workspaceId, afterSequence: historyRefreshSequence) + if page.retractionEpoch != historyRetractionEpoch { + page = try await store.listMutations(workspaceId: workspaceId, afterSequence: 0) + mutations = page.records + historyRefreshSequence = page.records.last?.sequence ?? 0 + historyRetractionEpoch = page.retractionEpoch + } else { + cachePersistedMutations(page.records) + historyRefreshSequence = max(historyRefreshSequence, page.records.last?.sequence ?? 0) + } + } + + func cachePersistedMutations(_ records: [Mutation]) { + var cachedSequences = Set(mutations.map(\.sequence)) + for record in records where cachedSequences.insert(record.sequence).inserted { + mutations.append(record) + } + mutations.sort { $0.sequence < $1.sequence } + } } diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index 5e24423..c2d4b6b 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -57,7 +57,12 @@ extension Workspace { struct ResolvedRevision: Sendable { var index: RevisionIndex - var load: ChangeSet.RevisionLoader + var readRange: @Sendable (WorkspacePath, UInt64, Int?) async throws -> Data + + var load: ChangeSet.RevisionLoader { + let readRange = readRange + return { path in try await readRange(path, 0, nil) } + } } func resolvedRevision(_ revision: Revision, currentSnapshot: Snapshot? = nil) async throws -> ResolvedRevision { @@ -71,11 +76,14 @@ extension Workspace { } return ResolvedRevision( index: RevisionIndex(snapshot: snapshot), - load: { path in + readRange: { path, offset, length in guard let entry = Self.snapshotEntry(at: path, in: snapshot.entry), case let .file(file) = entry else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } - return file.data + guard offset < UInt64(file.data.count) else { return Data() } + let start = Int(offset) + let count = min(length ?? file.data.count, file.data.count - start) + return Data(file.data.dropFirst(start).prefix(count)) } ) case let .checkpoint(id): @@ -93,13 +101,13 @@ extension Workspace { let snapshotID = checkpoint.snapshotId return ResolvedRevision( index: index, - load: { path in + readRange: { path, offset, length in guard let data = try await store.readSnapshotFile( id: snapshotID, workspaceId: workspaceID, path: path, - offset: 0, - length: nil + offset: offset, + length: length ) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } @@ -123,12 +131,7 @@ extension Workspace { } switch entry { case .file: - let data = try await resolved.load(current) - guard offset < UInt64(data.count) else { return Data() } - let start = data.index(data.startIndex, offsetBy: Int(offset)) - let end = length.map { data.index(start, offsetBy: $0, limitedBy: data.endIndex) ?? data.endIndex } - ?? data.endIndex - return Data(data[start.. Data { readCount += 1 - return try await base.readFile(path: path) + let data = try await base.readFile(path: path) + await capturePause?.pauseAfterReading(path) + return data } func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { @@ -175,6 +477,7 @@ private actor ReadCountingFileSystem: FileSystem { } func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await base.writeFile(path: path, data: data, append: append) + if cancelAfterWrite { withUnsafeCurrentTask { $0?.cancel() } } } func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) @@ -193,3 +496,33 @@ private actor ReadCountingFileSystem: FileSystem { try await base.setPermissions(path: path, permissions: permissions) } } + +private actor CaptureReadPause { + let path: WorkspacePath + private var armed = false + private var paused = false + private var pauseWaiters: [CheckedContinuation] = [] + private var resumeContinuation: CheckedContinuation? + + init(at path: WorkspacePath) { self.path = path } + + func arm() { armed = true } + + func pauseAfterReading(_ path: WorkspacePath) async { + guard armed, !paused, path == self.path else { return } + paused = true + for waiter in pauseWaiters { waiter.resume() } + pauseWaiters.removeAll() + await withCheckedContinuation { resumeContinuation = $0 } + } + + func waitUntilPaused() async { + if paused { return } + await withCheckedContinuation { pauseWaiters.append($0) } + } + + func resume() { + resumeContinuation?.resume() + resumeContinuation = nil + } +} From 316f1f456c84de2cb67e8f8d5fa5ccf12978a78c Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:25:36 -0700 Subject: [PATCH 07/34] Enforce filesystem limits atomically and fix in-memory, mount and watcher edge cases LimitedFileSystem serializes preflight and mutation through a FIFO gate, so concurrent writes can no longer jointly exceed a limit. Accounting comes from metadata (stat sizes, entry counts, file identity and link count) instead of content snapshots, counts dangling symlinks, charges hard-link aliases under a documented logical-bytes policy, resolves symlinks through the backend so mounts can't be used to bypass quotas, and lets an over-quota workspace shrink back under its limit. FileInfo gains optional identity and linkCount, and FileSystemFeatures gains replacementFollowsFinalSymlink, so wrappers can account exactly without a separate protocol. InMemoryFileSystem throws EISDIR instead of silently replacing a directory with a file, and its exists/glob use lstat semantics, so dangling symlinks survive snapshots and checkpoints. MountedFileSystem lists empty base directories, propagates backend errors, keeps stat consistent with listings, and includes nested mount points in directory listings. On Linux, recursive inotify watches include hidden directories, re-register replaced directories, retire watches in one batched pass (removing thousands of directories no longer stalls notifications), and treat the parent watch and unreadable subdirectories as best-effort. --- Sources/Workspace/FS/FileSystem.swift | 33 +- Sources/Workspace/FS/InMemoryFileSystem.swift | 79 ++- Sources/Workspace/FS/LimitedFileSystem.swift | 347 ++++++++++-- Sources/Workspace/FS/MountedFileSystem.swift | 126 ++--- .../Workspace/Support/FileChangeStreams.swift | 113 +++- .../FileSystemRegressionTests.swift | 523 ++++++++++++++++++ .../LinuxFileChangeStreamTests.swift | 170 ++++++ 7 files changed, 1240 insertions(+), 151 deletions(-) create mode 100644 Tests/WorkspaceTests/FileSystemRegressionTests.swift create mode 100644 Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index 3df5034..93a76a3 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -84,6 +84,12 @@ public struct FileInfo: Sendable, Codable { public var permissions: POSIXPermissions /// The last modification timestamp when available. public var modificationDate: Date? + /// Identity shared by aliases, when the backend supplies it. It must distinguish + /// different entries across backend instances and agree for the same underlying entry. + public var identity: FileIdentity? + /// Number of hard links to this entry, when known. One rules out additional hard links; + /// mounts may still expose the same entry at several virtual paths. + public var linkCount: UInt64? /// Creates metadata for a filesystem entry. public init( @@ -91,13 +97,31 @@ public struct FileInfo: Sendable, Codable { kind: FileTree.Kind, size: UInt64, permissions: POSIXPermissions, - modificationDate: Date? + modificationDate: Date?, + identity: FileIdentity? = nil, + linkCount: UInt64? = nil ) { self.path = path self.kind = kind self.size = size self.permissions = permissions self.modificationDate = modificationDate + self.identity = identity + self.linkCount = linkCount + } +} + +/// Opaque identity of a filesystem entry, shared by aliases across backend instances. +/// Device and inode values must distinguish different entries across instances and remain +/// stable during an entry's lifetime; replacements receive a new identity. Independent +/// in-memory backends need distinct device tokens; disk backends can use st_dev and st_ino. +public struct FileIdentity: Hashable, Sendable, Codable { + private let device: UInt64 + private let inode: UInt64 + + public init(device: UInt64, inode: UInt64) { + self.device = device + self.inode = inode } } @@ -134,6 +158,10 @@ public struct FileSystemFeatures: OptionSet, Sendable { public static let permissions = FileSystemFeatures(rawValue: 1 << 2) /// ``FileSystem/resolveRealPath(path:)`` resolves symlink chains. public static let realPathResolution = FileSystemFeatures(rawValue: 1 << 3) + /// Non-appending writes follow the final symlink instead of replacing the link itself. + /// Composing backends intersect this feature, so a mixed filesystem uses conservative + /// byte accounting where the flag is absent. Appends follow existing final symlinks. + public static let replacementFollowsFinalSymlink = FileSystemFeatures(rawValue: 1 << 4) } /// The low-level interface implemented by workspace storage backends. @@ -144,7 +172,8 @@ public protocol FileSystem: AnyObject, Sendable { func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] /// Reads the file contents at `path`. func readFile(path: WorkspacePath) async throws -> Data - /// Returns whether an entry exists at `path`. + /// Returns whether an entry exists at `path`, using lstat semantics: intermediate + /// symlinks are followed, but the final symlink itself exists even when its target does not. func exists(path: WorkspacePath) async -> Bool /// Expands a glob pattern relative to `currentDirectory`. func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index a9951db..d89e1b1 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -21,11 +21,14 @@ public actor InMemoryFileSystem: FileSystem { case symlink(String) } + let inode: UInt64 + var linkCount: UInt64 = 1 var kind: Kind var permissions: Int var modificationDate: Date - init(kind: Kind, permissions: Int, modificationDate: Date = Date()) { + init(inode: UInt64, kind: Kind, permissions: Int, modificationDate: Date = Date()) { + self.inode = inode self.kind = kind self.permissions = permissions self.modificationDate = modificationDate @@ -55,40 +58,26 @@ public actor InMemoryFileSystem: FileSystem { return 0 } } - - func clone() -> Node { - switch kind { - case let .file(data): - return Node(kind: .file(data), permissions: permissions, modificationDate: modificationDate) - case let .symlink(target): - return Node(kind: .symlink(target), permissions: permissions, modificationDate: modificationDate) - case let .directory(children): - var copiedChildren: [String: Node] = [:] - copiedChildren.reserveCapacity(children.count) - for (name, child) in children { - copiedChildren[name] = child.clone() - } - return Node(kind: .directory(copiedChildren), permissions: permissions, modificationDate: modificationDate) - } - } } private var root: Node + private let device = UInt64.random(in: 1...UInt64.max) + private var nextInode: UInt64 = 1 /// Creates an empty in-memory filesystem rooted at `/`. public init() { - root = Node(kind: .directory([:]), permissions: 0o755) + root = Node(inode: 0, kind: .directory([:]), permissions: 0o755) } /// See ``FileSystem/configure(rootDirectory:)``. func configure(rootDirectory: URL) async throws { _ = rootDirectory - root = Node(kind: .directory([:]), permissions: 0o755) + root = makeNode(kind: .directory([:]), permissions: 0o755) } /// Clears the filesystem and recreates an empty root directory. public func reset() { - root = Node(kind: .directory([:]), permissions: 0o755) + root = makeNode(kind: .directory([:]), permissions: 0o755) } /// See ``FileSystem/stat(path:)``. @@ -127,7 +116,7 @@ public actor InMemoryFileSystem: FileSystem { /// See ``FileSystem/capabilities()``. public func capabilities() async -> FileSystemFeatures { - [.symlinks, .hardLinks, .permissions, .realPathResolution] + [.symlinks, .hardLinks, .permissions, .realPathResolution, .replacementFollowsFinalSymlink] } /// See ``FileSystem/createFile(path:data:)``. Actor isolation makes the existence check and @@ -158,6 +147,10 @@ public actor InMemoryFileSystem: FileSystem { let (parent, name) = try parentDirectoryAndName(for: writePath) var children = try directoryChildren(of: parent) + if children[name]?.isDirectory == true { + throw posixError(EISDIR) + } + if append, let existing = children[name], case let .file(existingData) = existing.kind { @@ -165,7 +158,8 @@ public actor InMemoryFileSystem: FileSystem { existing.modificationDate = Date() children[name] = existing } else { - let node = Node(kind: .file(data), permissions: 0o644) + if let existing = children[name] { unlink(existing) } + let node = makeNode(kind: .file(data), permissions: 0o644) children[name] = node } @@ -201,7 +195,7 @@ public actor InMemoryFileSystem: FileSystem { throw posixError(ENOENT) } - let directory = Node(kind: .directory([:]), permissions: 0o755) + let directory = makeNode(kind: .directory([:]), permissions: 0o755) children[component] = directory current.kind = .directory(children) current.modificationDate = Date() @@ -226,6 +220,7 @@ public actor InMemoryFileSystem: FileSystem { var parentChildren = try directoryChildren(of: parent) parentChildren.removeValue(forKey: name) + unlink(entry) parent.kind = .directory(parentChildren) parent.modificationDate = Date() } @@ -289,7 +284,7 @@ public actor InMemoryFileSystem: FileSystem { throw posixError(EEXIST) } - destinationChildren[destinationName] = sourceNode.clone() + destinationChildren[destinationName] = clone(sourceNode) destinationParent.kind = .directory(destinationChildren) destinationParent.modificationDate = Date() } @@ -307,7 +302,7 @@ public actor InMemoryFileSystem: FileSystem { throw posixError(EEXIST) } - children[name] = Node(kind: .symlink(target), permissions: 0o777) + children[name] = makeNode(kind: .symlink(target), permissions: 0o777) parent.kind = .directory(children) parent.modificationDate = Date() } @@ -329,6 +324,7 @@ public actor InMemoryFileSystem: FileSystem { throw posixError(EEXIST) } + sourceNode.linkCount += 1 children[name] = sourceNode parent.kind = .directory(children) parent.modificationDate = Date() @@ -357,10 +353,10 @@ public actor InMemoryFileSystem: FileSystem { try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0) } - /// See ``FileSystem/exists(path:)``. + /// Returns entry existence without following the final symlink (lstat semantics). public func exists(path: WorkspacePath) async -> Bool { do { - _ = try node(at: path, followFinalSymlink: true) + _ = try node(at: path, followFinalSymlink: false) return true } catch { return false @@ -373,7 +369,7 @@ public actor InMemoryFileSystem: FileSystem { if !WorkspacePath.containsGlob(normalizedPattern.string) { let exists: Bool = { do { - _ = try node(at: normalizedPattern, followFinalSymlink: true) + _ = try node(at: normalizedPattern, followFinalSymlink: false) return true } catch { return false @@ -393,6 +389,29 @@ public actor InMemoryFileSystem: FileSystem { return matches.sorted() } + private func makeNode(kind: Node.Kind, permissions: Int, modificationDate: Date = Date()) -> Node { + let inode = nextInode + nextInode += 1 + return Node(inode: inode, kind: kind, permissions: permissions, modificationDate: modificationDate) + } + + private func clone(_ node: Node) -> Node { + let kind: Node.Kind + switch node.kind { + case let .file(data): kind = .file(data) + case let .symlink(target): kind = .symlink(target) + case let .directory(children): kind = .directory(children.mapValues { clone($0) }) + } + return makeNode(kind: kind, permissions: node.permissions, modificationDate: node.modificationDate) + } + + private func unlink(_ node: Node) { + if case let .directory(children) = node.kind { + for child in children.values { unlink(child) } + } + node.linkCount -= 1 + } + private func allVirtualPaths() -> [WorkspacePath] { var paths: [WorkspacePath] = [.root] collectPaths(node: root, currentPath: .root, into: &paths) @@ -427,7 +446,9 @@ public actor InMemoryFileSystem: FileSystem { kind: fileKind(for: node), size: node.size, permissions: POSIXPermissions(node.permissions), - modificationDate: node.modificationDate + modificationDate: node.modificationDate, + identity: FileIdentity(device: device, inode: node.inode), + linkCount: node.isDirectory ? nil : node.linkCount ) } diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index 8f058f2..1b71cda 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -1,6 +1,14 @@ import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + public struct FileSystemLimits: Sendable, Codable, Equatable { + /// Logical regular-file bytes: each path counts its size, including every hard-link alias. + /// Directories and symlinks contribute no bytes. public var maxTotalBytes: UInt64? public var maxEntryCount: Int? public var maxWriteBytes: Int? @@ -27,12 +35,27 @@ public enum FileSystemLimitError: Error, Sendable, Equatable, CustomStringConver } /// A serializing filesystem decorator that rejects mutations exceeding projected limits. +/// +/// All mutations through this instance share a FIFO gate. Mutations made directly to the base, +/// or through another decorator, are outside that gate. Accounting scans metadata, never content, +/// on each mutation so sequential external changes are reflected without a stale usage cache. +/// Append accounting uses metadata identities and link counts. Unknown aliases are charged +/// conservatively as all regular files of the target's size. Without the replacement feature, +/// non-appending writes are projected as replacing the final link itself; this is conservative +/// for bytes, but dangling-link entry counts require the backend to advertise its true behavior. +/// Custom backends must follow standard mutation semantics: replacement changes one entry; +/// append updates aliases; parents may be created. A composition cycle (for example a Mounted +/// base that mounts this same LimitedFileSystem) deadlocks while the gate is held. public actor LimitedFileSystem: FileSystem { private let base: any FileSystem + private let pathResolver: FileSystemPathResolver public let limits: FileSystemLimits + private var mutationActive = false + private var mutationWaiters: [CheckedContinuation] = [] public init(base: any FileSystem, limits: FileSystemLimits) { self.base = base + pathResolver = FileSystemPathResolver(base: base) self.limits = limits } @@ -59,86 +82,330 @@ public actor LimitedFileSystem: FileSystem { public func createFile(path: WorkspacePath, data: Data) async throws { try checkWrite(data.count) - try await preflight { try await $0.createFile(path: path, data: data) } - try await base.createFile(path: path, data: data) + try await mutate(.createFile(path: path, size: UInt64(data.count))) { + try await self.base.createFile(path: path, data: data) + } } public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try checkWrite(data.count) - try await preflight { try await $0.writeFile(path: path, data: data, append: append) } - try await base.writeFile(path: path, data: data, append: append) + try await mutate(.write(path: path, size: UInt64(data.count), append: append)) { + try await self.base.writeFile(path: path, data: data, append: append) + } } public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { - try await preflight { try await $0.createDirectory(path: path, recursive: recursive) } - try await base.createDirectory(path: path, recursive: recursive) + try await mutate(.directory(path: path)) { + try await self.base.createDirectory(path: path, recursive: recursive) + } } public func remove(path: WorkspacePath, recursive: Bool) async throws { - try await preflight { try await $0.remove(path: path, recursive: recursive) } - try await base.remove(path: path, recursive: recursive) + try await mutate(.remove(path: path)) { + try await self.base.remove(path: path, recursive: recursive) + } } public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { - try await preflight { try await $0.move(from: sourcePath, to: destinationPath) } - try await base.move(from: sourcePath, to: destinationPath) + try await mutate(.move(source: sourcePath, destination: destinationPath)) { + try await self.base.move(from: sourcePath, to: destinationPath) + } } public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { - try await preflight { - try await $0.copy(from: sourcePath, to: destinationPath, recursive: recursive) + try await mutate(.copy(source: sourcePath, destination: destinationPath)) { + try await self.base.copy(from: sourcePath, to: destinationPath, recursive: recursive) } - try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) } public func createSymlink(path: WorkspacePath, target: String) async throws { - try await preflight { try await $0.createSymlink(path: path, target: target) } - try await base.createSymlink(path: path, target: target) + try await mutate(.symlink(path: path)) { + try await self.base.createSymlink(path: path, target: target) + } } public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { - try await preflight { try await $0.createHardLink(path: path, target: target) } - try await base.createHardLink(path: path, target: target) + try await mutate(.hardLink(path: path, target: target)) { + try await self.base.createHardLink(path: path, target: target) + } } public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { + await acquireMutation() + defer { releaseMutation() } + try Task.checkCancellation() try await base.setPermissions(path: path, permissions: permissions) } + private func acquireMutation() async { + if !mutationActive { + mutationActive = true + return + } + await withCheckedContinuation { mutationWaiters.append($0) } + } + + private func releaseMutation() { + if mutationWaiters.isEmpty { + mutationActive = false + } else { + mutationWaiters.removeFirst().resume() + } + } + + private enum Mutation { + case createFile(path: WorkspacePath, size: UInt64) + case write(path: WorkspacePath, size: UInt64, append: Bool) + case directory(path: WorkspacePath) + case remove(path: WorkspacePath) + case move(source: WorkspacePath, destination: WorkspacePath) + case copy(source: WorkspacePath, destination: WorkspacePath) + case symlink(path: WorkspacePath) + case hardLink(path: WorkspacePath, target: WorkspacePath) + } + + private func mutate(_ mutation: Mutation, operation: () async throws -> Void) async throws { + await acquireMutation() + defer { releaseMutation() } + try Task.checkCancellation() + if limits.maxTotalBytes != nil || limits.maxEntryCount != nil { + try await preflight(mutation) + } + try await operation() + } + private func checkWrite(_ count: Int) throws { if let limit = limits.maxWriteBytes, count > limit { throw FileSystemLimitError.writeBytes(attempted: count, limit: limit) } } - private func preflight(_ mutation: @Sendable (InMemoryFileSystem) async throws -> Void) async throws { - let before = try await Snapshot.capture(from: base) - let scratch = InMemoryFileSystem() - try await Snapshot.restore(before, to: scratch) - try await mutation(scratch) - let projected = try await Snapshot.capture(from: scratch) - let usage = Self.usage(projected.entry, isRoot: true) - if let limit = limits.maxTotalBytes, usage.bytes > limit { - throw FileSystemLimitError.totalBytes(attempted: usage.bytes, limit: limit) + private struct Usage { + var bytes: UInt64 = 0 + var entries = 0 + + mutating func add(bytes amount: UInt64) { + let sum = bytes.addingReportingOverflow(amount) + bytes = sum.overflow ? UInt64.max : sum.partialValue + } + } + + private func preflight(_ mutation: Mutation) async throws { + var tree: [WorkspacePath: FileInfo] = [:] + var pending = [try await base.stat(path: .root)] + while let info = pending.popLast() { + let path = info.path + tree[path] = info + if info.kind == .directory { + for child in try await base.listDirectory(path: path) { + var metadata = child.info + metadata.path = path.appending(child.name) + pending.append(metadata) + } + } } - if let limit = limits.maxEntryCount, usage.entries > limit { - throw FileSystemLimitError.entryCount(attempted: usage.entries, limit: limit) + let current = Self.usage(tree) + var projected = current + let features = await base.capabilities() + + func resolve(_ path: WorkspacePath, in tree: [WorkspacePath: FileInfo], followFinal: Bool) async throws -> WorkspacePath { + try await pathResolver.resolve(path, in: tree, features: features, followFinal: followFinal) } + + func subtract(_ path: WorkspacePath) { + let removed = Self.usage(tree.filter { Self.contains($0.key, in: path) }) + projected.bytes -= removed.bytes + projected.entries -= removed.entries + } + func addParents(_ path: WorkspacePath) { + var parent = path.dirname + while !parent.isRoot { + if tree[parent] == nil { projected.entries += 1 } + parent = parent.dirname + } + } + func addLeaf(_ path: WorkspacePath, bytes: UInt64) { + subtract(path) + addParents(path) + if !path.isRoot { projected.entries += 1 } + projected.add(bytes: bytes) + } + + switch mutation { + case let .createFile(path, size): + let target = try await resolve(path, in: tree, followFinal: false) + if tree[target] != nil { throw Self.posix(EEXIST) } + addLeaf(target, bytes: size) + case let .write(path, size, append): + let entry = try await resolve(path, in: tree, followFinal: false) + let replaceLink = !append && tree[entry]?.kind == .symlink + && !features.contains(.replacementFollowsFinalSymlink) + let target = replaceLink ? entry : try await resolve(path, in: tree, followFinal: true) + if tree[target]?.kind == .directory { throw Self.posix(EISDIR) } + if append, let info = tree[target], info.kind == .file { + // Entry-only quotas need no alias accounting. Identity is already in the + // metadata walk, so disappearing candidates never cause another awaited stat. + if limits.maxTotalBytes != nil { + let aliases: UInt64 + if info.linkCount == 1, info.identity == nil { + aliases = 1 + } else { + aliases = UInt64(tree.reduce(0) { count, entry in + let candidate = entry.value + guard candidate.kind == .file, candidate.size == info.size else { return count } + if entry.key == target { return count + 1 } + if let identity = info.identity, let other = candidate.identity { + // A mount can expose even a single-link inode at multiple paths. + return identity == other ? count + 1 : count + } + if info.linkCount == 1 || candidate.linkCount == 1 { return count } + return count + 1 + }) + } + let growth = size.multipliedReportingOverflow(by: aliases) + projected.add(bytes: growth.overflow ? UInt64.max : growth.partialValue) + } + } else { + addLeaf(target, bytes: size) + } + case let .directory(path): + let target = try await resolve(path, in: tree, followFinal: true) + if tree[target] == nil { addLeaf(target, bytes: 0) } + case let .remove(path): + let target = try await resolve(path, in: tree, followFinal: false) + subtract(target) + case let .move(source, destination), let .copy(source, destination): + let from = try await resolve(source, in: tree, followFinal: false) + let to = try await resolve(destination, in: tree, followFinal: false) + guard tree[from] != nil else { throw Self.posix(ENOENT) } + if from == to { break } + let copied = Self.usage(tree.filter { Self.contains($0.key, in: from) }) + if case .move = mutation { + if tree[from]?.kind == .directory, Self.contains(to, in: from) { throw Self.posix(EINVAL) } + // Source and destination subtrees can overlap on an invalid backend move. + // Subtract their union once; let the backend preserve its destination error. + let removed = Self.usage(tree.filter { + Self.contains($0.key, in: to) || Self.contains($0.key, in: from) + }) + projected.bytes -= removed.bytes + projected.entries -= removed.entries + } else { + subtract(to) + } + addParents(to) + projected.add(bytes: copied.bytes) + projected.entries += copied.entries + (from.isRoot && !to.isRoot ? 1 : 0) + case let .symlink(path): + let target = try await resolve(path, in: tree, followFinal: false) + if tree[target] != nil { throw Self.posix(EEXIST) } + addLeaf(target, bytes: 0) + case let .hardLink(path, source): + let target = try await resolve(path, in: tree, followFinal: false) + let from = try await resolve(source, in: tree, followFinal: false) + if tree[target] != nil { throw Self.posix(EEXIST) } + guard let info = tree[from] else { throw Self.posix(ENOENT) } + if info.kind == .directory { throw Self.posix(EPERM) } + addLeaf(target, bytes: info.kind == .file ? info.size : 0) + } + if let limit = limits.maxTotalBytes, projected.bytes > limit, projected.bytes > current.bytes { + throw FileSystemLimitError.totalBytes(attempted: projected.bytes, limit: limit) + } + if let limit = limits.maxEntryCount, projected.entries > limit, projected.entries > current.entries { + throw FileSystemLimitError.entryCount(attempted: projected.entries, limit: limit) + } + } + + private static func usage(_ tree: [WorkspacePath: FileInfo]) -> Usage { + var result = Usage() + for (path, info) in tree { + if !path.isRoot { result.entries += 1 } + if info.kind == .file { result.add(bytes: info.size) } + } + return result } - private static func usage(_ entry: Snapshot.Entry, isRoot: Bool) -> (bytes: UInt64, entries: Int) { - switch entry { - case .missing: - return (0, 0) - case let .file(file): - return (UInt64(file.data.count), isRoot ? 0 : 1) - case .symlink: - return (0, isRoot ? 0 : 1) - case let .directory(directory): - return directory.children.reduce((bytes: 0, entries: isRoot ? 0 : 1)) { result, child in - let childUsage = usage(child, isRoot: false) - return (result.bytes + childUsage.bytes, result.entries + childUsage.entries) + private static func contains(_ path: WorkspacePath, in parent: WorkspacePath) -> Bool { + parent.isRoot || path == parent || path.string.hasPrefix(parent.string + "/") + } + + private static func posix(_ code: Int32) -> NSError { + NSError(domain: NSPOSIXErrorDomain, code: Int(code)) + } +} + +// Resolves in the backend's own namespace. A Mounted backend owns the interpretation of +// absolute symlink targets; only it can translate a dangling target back to a virtual path. +// Only a directly held MountedFileSystem is special-cased. When another decorator wraps +// MountedFileSystem, dangling links inside a mount fall back to virtual-namespace resolution. +struct FileSystemPathResolver: Sendable { + private let base: any FileSystem + private let mounted: MountedFileSystem? + + init(base: any FileSystem) { + self.base = base + mounted = base as? MountedFileSystem + } + + func resolve( + _ path: WorkspacePath, in tree: [WorkspacePath: FileInfo]? = nil, + features: FileSystemFeatures, followFinal: Bool, depth: Int = 0 + ) async throws -> WorkspacePath { + guard depth < 64 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } + if !followFinal, !path.isRoot { + return try await resolve(path.dirname, in: tree, features: features, followFinal: true, depth: depth) + .appending(path.basename) + } + if features.contains(.realPathResolution) { + var ancestor = path + var missing: [String] = [] + while true { + do { + let real = try await base.resolveRealPath(path: ancestor) + return missing.isEmpty ? real : real.appending(missing.reversed().joined(separator: "/")) + } catch { + guard Self.isMissing(error), !ancestor.isRoot else { throw error } + // The ancestor may still contain intermediate symlinks, while the tree + // uses lstat paths. Ask the backend to resolve those intermediate links. + let info = try? await base.stat(path: ancestor) + if info?.kind == .symlink { + if let mounted { return try await mounted.resolvePotentialPath(path: path) } + let target = try await base.readSymlink(path: ancestor) + let targetPath = try WorkspacePath(validating: target, relativeTo: ancestor.dirname) + let combined = missing.isEmpty ? targetPath : targetPath.appending(missing.reversed().joined(separator: "/")) + return try await resolve(combined, in: tree, features: features, followFinal: true, depth: depth + 1) + } + missing.append(ancestor.basename) + ancestor = ancestor.dirname + } + } + } + if let mounted { return try await mounted.resolvePotentialPath(path: path) } + var current = WorkspacePath.root + for (index, component) in path.components.enumerated() { + current = current.appending(component) + let info: FileInfo? + if let tree { info = tree[current] } + else { info = try? await base.stat(path: current) } + if info?.kind == .symlink { + let target = try await base.readSymlink(path: current) + let targetPath = try WorkspacePath(validating: target, relativeTo: current.dirname) + let suffix = path.components.dropFirst(index + 1).joined(separator: "/") + return try await resolve( + suffix.isEmpty ? targetPath : targetPath.appending(suffix), in: tree, + features: features, followFinal: true, depth: depth + 1 + ) + } + if index < path.components.count - 1, let info, info.kind != .directory { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) } } + return current + } + + private static func isMissing(_ error: Error) -> Bool { + let error = error as NSError + return (error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT)) + || (error.domain == NSCocoaErrorDomain + && [NSFileNoSuchFileError, NSFileReadNoSuchFileError].contains(error.code)) } } diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index 75178d1..c2adeab 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -78,71 +78,54 @@ public final class MountedFileSystem: FileSystem, @unchecked Sendable { /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { let mounts = mountsSnapshot() - if let resolved = resolveMounted(path: path, mounts: mounts) { - var info = try await resolved.filesystem.stat(path: resolved.relativePath) - info.path = path - return info - } - - if hasSyntheticDirectory(at: path, mounts: mounts) { - return FileInfo( - path: path, - kind: .directory, - size: 0, - permissions: .defaultDirectory, - modificationDate: nil + let resolved = resolveWritable(path: path, mounts: mounts) + do { + return Self.virtualInfo( + try await resolved.filesystem.stat(path: resolved.relativePath), + at: path ) + } catch { + guard Self.isMissing(error), + path.isRoot || !syntheticChildMountNames(under: path, mounts: mounts).isEmpty else { throw error } + return Self.syntheticInfo(at: path) } - - return try await base.stat(path: path) } /// See ``FileSystem/listDirectory(path:)``. public func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { let mounts = mountsSnapshot() - if let resolved = resolveMounted(path: path, mounts: mounts) { - let entries = try await resolved.filesystem.listDirectory(path: resolved.relativePath) - return entries.map { entry in - DirectoryEntry( + let syntheticNames = syntheticChildMountNames(under: path, mounts: mounts) + let resolved = resolveWritable(path: path, mounts: mounts) + var merged: [String: DirectoryEntry] = [:] + do { + for entry in try await resolved.filesystem.listDirectory(path: resolved.relativePath) { + merged[entry.name] = DirectoryEntry( name: entry.name, - info: FileInfo( - path: path.appending(entry.name), - kind: entry.info.kind, - size: entry.info.size, - permissions: entry.info.permissions, - modificationDate: entry.info.modificationDate - ) + info: Self.virtualInfo(entry.info, at: path.appending(entry.name)) ) } - } - - var merged: [String: DirectoryEntry] = [:] - let baseHasPath = path.isRoot ? true : await base.exists(path: path) - if baseHasPath { - if let baseEntries = try? await base.listDirectory(path: path) { - for entry in baseEntries { - merged[entry.name] = entry - } + } catch { + let nsError = error as NSError + guard Self.isMissing(error) + || (nsError.domain == NSPOSIXErrorDomain && nsError.code == Int(ENOTDIR)) else { throw error } + guard path.isRoot || !syntheticNames.isEmpty else { throw error } + let listingError = error + var missing = false + do { + _ = try await resolved.filesystem.stat(path: resolved.relativePath) + } catch { + guard Self.isMissing(error) else { throw listingError } + missing = true } + guard missing else { throw listingError } } - for syntheticName in syntheticChildMountNames(under: path, mounts: mounts) { - merged[syntheticName] = DirectoryEntry( - name: syntheticName, - info: FileInfo( - path: path.appending(syntheticName), - kind: .directory, - size: 0, - permissions: .defaultDirectory, - modificationDate: nil - ) - ) + for name in syntheticNames { + let child = path.appending(name) + // Actual mounts shadow entries. Synthetic ancestors use backend metadata if + // present, keeping traversal consistent with stat (including regular files). + merged[name] = DirectoryEntry(name: name, info: try await stat(path: child)) } - - if merged.isEmpty, !hasSyntheticDirectory(at: path, mounts: mounts) { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) - } - return merged.values.sorted { $0.name < $1.name } } @@ -297,16 +280,19 @@ public final class MountedFileSystem: FileSystem, @unchecked Sendable { return resolved.mountPoint.isRoot ? real : WorkspacePath.join(resolved.mountPoint, String(real.string.dropFirst())) } + // Used only when a symlink target or suffix is missing. Resolve inside the selected + // backend before translating back; raw absolute targets are never outer virtual paths. + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + let resolved = resolveWritable(path: path, mounts: mountsSnapshot()) + let real = try await FileSystemPathResolver(base: resolved.filesystem).resolve( + resolved.relativePath, features: await resolved.filesystem.capabilities(), followFinal: true + ) + return resolved.mountPoint.isRoot ? real : WorkspacePath.join(resolved.mountPoint, String(real.string.dropFirst())) + } + /// See ``FileSystem/exists(path:)``. public func exists(path: WorkspacePath) async -> Bool { - let mounts = mountsSnapshot() - if let resolved = resolveMounted(path: path, mounts: mounts) { - return await resolved.filesystem.exists(path: resolved.relativePath) - } - if hasSyntheticDirectory(at: path, mounts: mounts) { - return true - } - return await base.exists(path: path) + (try? await stat(path: path)) != nil } /// See ``FileSystem/glob(pattern:currentDirectory:)``. @@ -383,9 +369,21 @@ public final class MountedFileSystem: FileSystem, @unchecked Sendable { return Array(Set(paths)) } - private func hasSyntheticDirectory(at path: WorkspacePath, mounts: [Mount]) -> Bool { - path.isRoot || mounts.contains { parentPath(of: $0.mountPoint) == path } - || syntheticChildMountNames(under: path, mounts: mounts).isEmpty == false + private static func virtualInfo(_ info: FileInfo, at path: WorkspacePath) -> FileInfo { + var adjusted = info + adjusted.path = path + return adjusted + } + + private static func syntheticInfo(at path: WorkspacePath) -> FileInfo { + FileInfo(path: path, kind: .directory, size: 0, permissions: .defaultDirectory, modificationDate: nil) + } + + private static func isMissing(_ error: Error) -> Bool { + let error = error as NSError + return (error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT)) + || (error.domain == NSCocoaErrorDomain + && [NSFileNoSuchFileError, NSFileReadNoSuchFileError].contains(error.code)) } private func syntheticChildMountNames(under path: WorkspacePath, mounts: [Mount]) -> [String] { @@ -405,10 +403,6 @@ public final class MountedFileSystem: FileSystem, @unchecked Sendable { return names.sorted() } - private func parentPath(of path: WorkspacePath) -> WorkspacePath { - path.dirname - } - private func isPath(_ candidate: WorkspacePath, inside parent: WorkspacePath) -> Bool { if parent.isRoot { return candidate.string.hasPrefix("/") && !candidate.isRoot diff --git a/Sources/Workspace/Support/FileChangeStreams.swift b/Sources/Workspace/Support/FileChangeStreams.swift index 20f25cb..de02f91 100644 --- a/Sources/Workspace/Support/FileChangeStreams.swift +++ b/Sources/Workspace/Support/FileChangeStreams.swift @@ -53,7 +53,10 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { private let continuation: AsyncStream.Continuation private let descriptor: Int32 private let source: DispatchSourceRead - private var watchesByPath: [String: Int32] = [:] + private var pathsByWatch: [Int32: Set] = [:] + // Keep the root's parent watched so recreating the root can restore coverage even + // when it was absent during the IN_DELETE_SELF/IN_MOVE_SELF rescan. + private var parentWatch: Int32? init( directory: URL, @@ -65,15 +68,19 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { self.continuation = continuation descriptor = inotify_init1(Int32(IN_NONBLOCK | IN_CLOEXEC)) guard descriptor >= 0 else { throw POSIXError(.init(rawValue: errno) ?? .EIO) } - source = DispatchSource.makeReadSource(fileDescriptor: descriptor, queue: .global(qos: .utility)) + source = DispatchSource.makeReadSource( + fileDescriptor: descriptor, queue: DispatchQueue(label: "Workspace.inotify", qos: .utility) + ) + source.setCancelHandler { [descriptor] in close(descriptor) } do { - try addMissingWatches() + try addMissingWatches(requireRoot: true) } catch { - close(descriptor) + // A dispatch source must be resumed before destruction, even on setup failure. + source.cancel() + source.resume() throw error } source.setEventHandler { [weak self] in self?.handleEvents() } - source.setCancelHandler { [descriptor] in close(descriptor) } source.resume() } @@ -82,19 +89,73 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { private func handleEvents() { var buffer = [UInt8](repeating: 0, count: 16 * 1024) var received = false - while read(descriptor, &buffer, buffer.count) > 0 { received = true } + var retired = Set() + var ignored = Set() + var overflow = false + // Drain first, retire each descriptor once, then reconcile against the live tree. + // No event performs a scan of the path cache or a scan of descendant prefixes. + while true { + let count = read(descriptor, &buffer, buffer.count) + if count < 0, errno == EINTR { continue } + guard count > 0 else { break } + buffer.withUnsafeBytes { bytes in + var offset = 0 + let headerSize = MemoryLayout.size + while offset + headerSize <= count { + let event = bytes.loadUnaligned(fromByteOffset: offset, as: inotify_event.self) + let recordSize = headerSize + Int(event.len) + guard recordSize <= count - offset else { break } + defer { offset += recordSize } + if event.wd == parentWatch { + if event.mask & UInt32(IN_IGNORED) != 0 { + parentWatch = nil + received = true + } else { + let nameBytes = bytes[(offset + headerSize)..<(offset + recordSize)].prefix { $0 != 0 } + guard String(decoding: nameBytes, as: UTF8.self) == directory.lastPathComponent else { continue } + } + } + received = true + if event.mask & UInt32(IN_IGNORED | IN_DELETE_SELF | IN_MOVE_SELF) != 0 { + retired.insert(event.wd) + if event.mask & UInt32(IN_IGNORED) != 0 { ignored.insert(event.wd) } + } + if event.mask & UInt32(IN_Q_OVERFLOW) != 0 { overflow = true } + } + } + } guard received else { return } - try? addMissingWatches() + if overflow { + var watches = Set(pathsByWatch.keys) + if let parentWatch { watches.insert(parentWatch) } + for watch in watches where !ignored.contains(watch) { _ = inotify_rm_watch(descriptor, watch) } + pathsByWatch.removeAll(keepingCapacity: true) + parentWatch = nil + } else { + for watch in retired { + if pathsByWatch.removeValue(forKey: watch) != nil { + if !ignored.contains(watch) { _ = inotify_rm_watch(descriptor, watch) } + } + } + } + do { + try addMissingWatches(requireRoot: false) + } catch { + continuation.finish() + cancel() + return + } continuation.yield() } - private func addMissingWatches() throws { + private func addMissingWatches(requireRoot: Bool) throws { let paths: [String] if recursive { let enumerator = FileManager.default.enumerator( at: directory, includingPropertiesForKeys: [.isDirectoryKey], - options: [.skipsHiddenFiles] + options: [], + errorHandler: { _, _ in true } ) var discovered = [directory.path] while let url = enumerator?.nextObject() as? URL { @@ -106,15 +167,39 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { } else { paths = [directory.path] } - watchesByPath = watchesByPath.filter { FileManager.default.fileExists(atPath: $0.key) } let mask = UInt32( IN_CREATE | IN_DELETE | IN_CLOSE_WRITE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB - | IN_DELETE_SELF | IN_MOVE_SELF + | IN_DELETE_SELF | IN_MOVE_SELF | IN_ONLYDIR ) - for path in paths where watchesByPath[path] == nil { + var nextByWatch: [Int32: Set] = [:] + for path in paths { + // The kernel returns the current wd for the same inode and a new wd for a + // replacement. Re-register every discovered path, rather than trusting its name. let watch = path.withCString { inotify_add_watch(descriptor, $0, mask) } - guard watch >= 0 else { throw POSIXError(.init(rawValue: errno) ?? .EIO) } - watchesByPath[path] = watch + if watch < 0 { + let code = errno + if path == directory.path, requireRoot || (code != ENOENT && code != ENOTDIR) { + throw POSIXError(.init(rawValue: code) ?? .EIO) + } + // Children can disappear, become files, or deny reads during enumeration. + // Skip them and keep registering later siblings; future events will retry. + continue + } + nextByWatch[watch, default: []].insert(path) + } + // A descriptor can serve multiple paths. Remove it only after every live path has + // been registered, and only if the new reverse map no longer references that wd. + for watch in pathsByWatch.keys where nextByWatch[watch] == nil { + _ = inotify_rm_watch(descriptor, watch) + } + pathsByWatch = nextByWatch + if parentWatch == nil, directory.path != "/" { + let watch = directory.deletingLastPathComponent().path.withCString { + inotify_add_watch(descriptor, $0, UInt32(IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_ONLYDIR)) + } + // The parent may grant traversal without read access (for example 0711). + // Its watch improves root-recreation coverage but cannot disable a valid root. + if watch >= 0 { parentWatch = watch } } } } diff --git a/Tests/WorkspaceTests/FileSystemRegressionTests.swift b/Tests/WorkspaceTests/FileSystemRegressionTests.swift new file mode 100644 index 0000000..c87aabd --- /dev/null +++ b/Tests/WorkspaceTests/FileSystemRegressionTests.swift @@ -0,0 +1,523 @@ +import Foundation +import Testing +@testable import Workspace + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +@Suite("Filesystem regression contracts") +struct FileSystemRegressionTests { + @Test + func `concurrent writes cannot share the same remaining quota`() async throws { + for _ in 0..<20 { + let memory = InMemoryFileSystem() + let limited = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 5)) + let successes = try await withThrowingTaskGroup(of: Bool.self) { group in + for path: WorkspacePath in ["/a", "/b"] { + group.addTask { + do { + try await limited.writeFile(path: path, data: Data("1234".utf8), append: false) + return true + } catch let error as FileSystemLimitError { + #expect(error == .totalBytes(attempted: 8, limit: 5)) + return false + } + } + } + var count = 0 + for try await success in group where success { count += 1 } + return count + } + #expect(successes == 1) + #expect(try await memory.listDirectory(path: "/").count == 1) + // A rejected preflight must release the gate for the next mutation. + for path: WorkspacePath in ["/a", "/b"] { try await limited.remove(path: path, recursive: false) } + try await limited.createFile(path: "/after", data: Data("12345".utf8)) + } + } + + @Test + func `quota gate spans an awaited backend mutation`() async throws { + let memory = InMemoryFileSystem() + let latch = RegressionMutationLatch() + let backend = RegressionBackend(memory: memory, writeLatch: latch) + let limited = LimitedFileSystem(base: backend, limits: .init(maxTotalBytes: 5)) + let first = Task { try await limited.writeFile(path: "/first", data: Data("1234".utf8), append: false) } + await latch.waitUntilEntered() + let second = Task { () -> Bool in + do { + try await limited.writeFile(path: "/second", data: Data("1234".utf8), append: false) + return true + } catch let error as FileSystemLimitError { + #expect(error == .totalBytes(attempted: 8, limit: 5)) + return false + } + } + for _ in 0..<10 { await Task.yield() } + await latch.release() + try await first.value + #expect(try await second.value == false) + #expect(try await memory.listDirectory(path: "/").map(\.name) == ["first"]) + } + + @Test + func `dangling symlinks count as entries but not regular file bytes`() async throws { + let memory = InMemoryFileSystem() + let none = LimitedFileSystem(base: memory, limits: .init(maxEntryCount: 0)) + await #expect(throws: FileSystemLimitError.entryCount(attempted: 1, limit: 0)) { + try await none.createSymlink(path: "/dangling", target: "/missing") + } + #expect(!(await memory.exists(path: "/dangling"))) + let one = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 0, maxEntryCount: 1)) + try await one.createSymlink(path: "/dangling", target: "/missing") + await #expect(throws: FileSystemLimitError.entryCount(attempted: 2, limit: 1)) { + try await one.createSymlink(path: "/another", target: "/missing") + } + await #expect(throws: FileSystemLimitError.entryCount(attempted: 2, limit: 1)) { + try await one.createHardLink(path: "/alias", target: "/dangling") + } + } + + @Test + func `append charges every hard link while replacement charges one entry`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/file", data: Data("ab".utf8), append: false) + try await memory.createHardLink(path: "/alias", target: "/file") + let limited = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 5)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 6, limit: 5)) { + try await limited.writeFile(path: "/alias", data: Data("c".utf8), append: true) + } + #expect(try await memory.readFile(path: "/file") == Data("ab".utf8)) + try await limited.writeFile(path: "/alias", data: Data("xyz".utf8), append: false) + #expect(try await memory.readFile(path: "/file") == Data("ab".utf8)) + #expect(try await memory.readFile(path: "/alias") == Data("xyz".utf8)) + try await limited.remove(path: "/file", recursive: false) + try await limited.writeFile(path: "/alias", data: Data("12".utf8), append: true) + #expect(try await memory.stat(path: "/alias").size == 5) + } + + @Test + func `identity distinguishes unrelated files with matching metadata sizes`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/file", data: Data("ab".utf8), append: false) + try await memory.writeFile(path: "/unrelated", data: Data("ab".utf8), append: false) + let mounted = MountedFileSystem(mounts: [.init(mountPoint: "/", fileSystem: memory)]) + let limited = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 5)) + try await limited.writeFile(path: "/file", data: Data("c".utf8), append: true) + #expect(try await memory.readFile(path: "/unrelated") == Data("ab".utf8)) + } + + @Test + func `a known single link permits exact appends through metadata only wrappers`() async throws { + let memory = InMemoryFileSystem() + for index in 0..<20 { + try await memory.writeFile(path: WorkspacePath(normalizing: "/f\(index)"), data: Data(), append: false) + } + let backend = RegressionBackend(memory: memory, denyChildStats: true, omitIdentities: true) + let limited = LimitedFileSystem(base: backend, limits: .init(maxTotalBytes: 5)) + try await limited.writeFile(path: "/f0", data: Data(count: 5), append: true) + #expect(try await memory.stat(path: "/f0").size == 5) + #expect(try await memory.stat(path: "/f1").size == 0) + } + + @Test + func `appending through distinct backends sharing an entry charges every exposed path`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/file", data: Data(count: 2), append: false) + let mounted = MountedFileSystem(mounts: [ + .init(mountPoint: "/a", fileSystem: RegressionBackend(memory: memory)), + .init(mountPoint: "/b", fileSystem: RegressionBackend(memory: memory)), + ]) + let identity = try await memory.stat(path: "/file").identity + #expect(identity != nil) + #expect(try await mounted.stat(path: "/a/file").identity == identity) + #expect(try await mounted.stat(path: "/b/file").identity == identity) + #expect(try await mounted.listDirectory(path: "/a").first?.info.identity == identity) + #expect(try await mounted.listDirectory(path: "/b").first?.info.identity == identity) + let limited = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 5)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 6, limit: 5)) { + try await limited.writeFile(path: "/a/file", data: Data(count: 1), append: true) + } + #expect(try await memory.stat(path: "/file").size == 2) + } + + @Test + func `hard link metadata tracks copies replacements moves and recursive unlinking`() async throws { + let memory = InMemoryFileSystem() + try await memory.createDirectory(path: "/dir", recursive: false) + try await memory.writeFile(path: "/file", data: Data(count: 2), append: false) + try await memory.createHardLink(path: "/dir/alias", target: "/file") + try await memory.createHardLink(path: "/dir/second", target: "/file") + let original = try await memory.stat(path: "/file") + #expect(original.identity != nil) + #expect(original.linkCount == 3) + #expect(try await memory.stat(path: "/dir/alias").identity == original.identity) + try await memory.copy(from: "/dir", to: "/copy", recursive: true) + #expect(try await memory.stat(path: "/copy/alias").identity != original.identity) + #expect(try await memory.stat(path: "/copy/alias").linkCount == 1) + try await memory.move(from: "/file", to: "/renamed") + #expect(try await memory.stat(path: "/renamed").identity == original.identity) + try await memory.remove(path: "/dir", recursive: true) + #expect(try await memory.stat(path: "/renamed").linkCount == 1) + try await memory.createHardLink(path: "/alias", target: "/renamed") + try await memory.writeFile(path: "/alias", data: Data(count: 3), append: false) + #expect(try await memory.stat(path: "/alias").identity != original.identity) + #expect(try await memory.stat(path: "/renamed").linkCount == 1) + let other = InMemoryFileSystem() + try await other.writeFile(path: "/file", data: Data(count: 2), append: false) + #expect(try await other.stat(path: "/file").identity != original.identity) + await memory.reset() + try await memory.writeFile(path: "/file", data: Data(count: 2), append: false) + #expect(try await memory.stat(path: "/file").identity != original.identity) + } + + @Test + func `FileInfo decodes older metadata without identity or link count`() throws { + let legacy = Data(#"{"path":"/file","kind":"file","size":4,"permissions":{"rawValue":420}}"#.utf8) + let info = try JSONDecoder().decode(FileInfo.self, from: legacy) + #expect(info.path == "/file") + #expect(info.size == 4) + #expect(info.identity == nil) + #expect(info.linkCount == nil) + } + + @Test + func `unknown identities conservatively protect preexisting disk aliases`() async throws { + let root = try TestSupport.temporaryDirectory("LimitedDiskAliases") + defer { TestSupport.remove(root) } + let disk = try LocalFileSystem(root: root) + try await disk.writeFile(path: "/file", data: Data("ab".utf8), append: false) + try await disk.createHardLink(path: "/alias", target: "/file") + let limited = LimitedFileSystem(base: disk, limits: .init(maxTotalBytes: 5)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 6, limit: 5)) { + try await limited.writeFile(path: "/alias", data: Data("c".utf8), append: true) + } + #expect(try await disk.readFile(path: "/file") == Data("ab".utf8)) + try await limited.writeFile(path: "/alias", data: Data("abc".utf8), append: false) + #expect(try await disk.stat(path: "/file").size == 2) + } + + @Test + func `limit preflight uses metadata without reading file content`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/large", data: Data("1234".utf8), append: false) + let backend = RegressionBackend(memory: memory, denyReads: true, denyChildStats: true) + let limited = LimitedFileSystem(base: backend, limits: .init(maxTotalBytes: 8, maxEntryCount: 3)) + try await limited.createDirectory(path: "/dir", recursive: false) + try await limited.copy(from: "/large", to: "/dir/copy", recursive: false) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 9, limit: 8)) { + try await limited.writeFile(path: "/new", data: Data([1]), append: false) + } + try await limited.move(from: "/dir/copy", to: "/copy") + try await limited.remove(path: "/dir", recursive: true) + try await limited.writeFile(path: "/large", data: Data([1]), append: false) + #expect(try await memory.readFile(path: "/copy") == Data("1234".utf8)) + } + + @Test + func `limited moves preserve existing destination errors for overlapping paths`() async throws { + let memory = InMemoryFileSystem() + try await memory.createDirectory(path: "/dir", recursive: false) + try await memory.writeFile(path: "/dir/file", data: Data(count: 4), append: false) + let limited = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 8, maxEntryCount: 3)) + do { + try await limited.move(from: "/dir/file", to: "/dir") + Issue.record("expected existing destination rejection") + } catch { #expect((error as NSError).code == Int(EEXIST)) } + #expect(try await memory.stat(path: "/dir/file").size == 4) + try await limited.move(from: "/dir/file", to: "/after") + #expect(try await memory.stat(path: "/after").size == 4) + } + + @Test + func `dangling link through a symlinked parent charges the created target entry`() async throws { + let memory = InMemoryFileSystem() + try await memory.createDirectory(path: "/dir", recursive: false) + try await memory.createSymlink(path: "/dir/dangle", target: "/target") + try await memory.createSymlink(path: "/link", target: "/dir") + let limited = LimitedFileSystem(base: memory, limits: .init(maxEntryCount: 3)) + await #expect(throws: FileSystemLimitError.entryCount(attempted: 4, limit: 3)) { + try await limited.writeFile(path: "/link/dangle", data: Data([1]), append: false) + } + #expect(!(await memory.exists(path: "/target"))) + } + + @Test + func `mounted symlink writes are charged in the backend namespace`() async throws { + let base = InMemoryFileSystem() + try await base.writeFile(path: "/data", data: Data(count: 1000), append: false) + let inner = InMemoryFileSystem() + try await inner.createSymlink(path: "/link", target: "/data") + let mounted = MountedFileSystem(base: base, mounts: [.init(mountPoint: "/m", fileSystem: inner)]) + let limited = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 1100)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 1900, limit: 1100)) { + try await limited.writeFile(path: "/m/link", data: Data(count: 900), append: false) + } + #expect(!(await inner.exists(path: "/data"))) + #expect(try await base.stat(path: "/data").size == 1000) + + try await inner.createDirectory(path: "/dir", recursive: false) + try await inner.createSymlink(path: "/parent", target: "/dir") + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 1900, limit: 1100)) { + try await limited.writeFile(path: "/m/parent/new", data: Data(count: 900), append: false) + } + #expect(!(await inner.exists(path: "/dir/new"))) + } + + @Test + func `over quota mutations may reduce or preserve usage but cannot grow it`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/large", data: Data(count: 10), append: false) + try await memory.writeFile(path: "/small", data: Data(count: 2), append: false) + let limited = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 5, maxEntryCount: 0)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 13, limit: 5)) { + try await limited.writeFile(path: "/large", data: Data(count: 1), append: true) + } + try await limited.move(from: "/small", to: "/renamed") + try await limited.remove(path: "/renamed", recursive: false) + #expect(try await memory.stat(path: "/large").size == 10) + try await limited.writeFile(path: "/large", data: Data(count: 6), append: false) + try await limited.remove(path: "/large", recursive: false) + #expect(try await memory.listDirectory(path: "/").isEmpty) + } + + @Test + func `replacement accounting respects final symlink write behavior`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/file", data: Data("ab".utf8), append: false) + try await memory.createSymlink(path: "/link", target: "/file") + let memoryLimited = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 3)) + try await memoryLimited.writeFile(path: "/link", data: Data("xyz".utf8), append: false) + #expect(try await memory.readFile(path: "/file") == Data("xyz".utf8)) + #expect(try await memory.stat(path: "/link").kind == .symlink) + + let root = try TestSupport.temporaryDirectory("LimitedSymlinkReplacement") + defer { TestSupport.remove(root) } + let disk = try LocalFileSystem(root: root) + try await disk.writeFile(path: "/file", data: Data("ab".utf8), append: false) + try await disk.createSymlink(path: "/link", target: "/file") + let diskLimited = LimitedFileSystem(base: disk, limits: .init(maxTotalBytes: 3)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 5, limit: 3)) { + try await diskLimited.writeFile(path: "/link", data: Data("xyz".utf8), append: false) + } + #expect(try await disk.stat(path: "/link").kind == .symlink) + #expect(try await disk.readFile(path: "/file") == Data("ab".utf8)) + } + + @Test + func `write accounting follows symlink parents and counts created disk parents`() async throws { + let memory = InMemoryFileSystem() + try await memory.createDirectory(path: "/dir", recursive: false) + try await memory.writeFile(path: "/dir/file", data: Data("1234".utf8), append: false) + try await memory.createSymlink(path: "/link", target: "/dir") + let limited = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 5)) + try await limited.writeFile(path: "/link/file", data: Data("abcde".utf8), append: false) + #expect(try await memory.readFile(path: "/dir/file") == Data("abcde".utf8)) + let root = try TestSupport.temporaryDirectory("LimitedDiskParents") + defer { TestSupport.remove(root) } + let disk = try LocalFileSystem(root: root) + let entries = LimitedFileSystem(base: disk, limits: .init(maxEntryCount: 2)) + await #expect(throws: FileSystemLimitError.entryCount(attempted: 3, limit: 2)) { + try await entries.writeFile(path: "/one/two/file", data: Data(), append: false) + } + #expect(!(await disk.exists(path: "/one"))) + } + + @Test(arguments: [false, true]) + func `writing a directory preserves its children and throws EISDIR`(append: Bool) async throws { + let memory = InMemoryFileSystem() + try await memory.createDirectory(path: "/dir", recursive: false) + try await memory.writeFile(path: "/dir/important", data: Data("keep".utf8), append: false) + try await memory.createSymlink(path: "/link", target: "/dir") + for path: WorkspacePath in ["/dir", "/link"] { + do { + try await memory.writeFile(path: path, data: Data("oops".utf8), append: append) + Issue.record("expected EISDIR") + } catch { + #expect((error as NSError).domain == NSPOSIXErrorDomain) + #expect((error as NSError).code == Int(EISDIR)) + } + } + #expect(try await memory.readFile(path: "/dir/important") == Data("keep".utf8)) + #expect(try await memory.stat(path: "/dir").kind == .directory) + } + + @Test + func `dangling links exist in memory glob snapshots and checkpoints`() async throws { + let memory = InMemoryFileSystem() + try await memory.createSymlink(path: "/dangling", target: "/missing") + #expect(await memory.exists(path: "/dangling")) + #expect(try await memory.glob(pattern: "/dangling", currentDirectory: "/") == ["/dangling"]) + #expect(!(await memory.exists(path: "/dangling/child"))) + let snapshot = try await Snapshot.capture(from: memory) + let restored = InMemoryFileSystem() + try await Snapshot.restore(snapshot, to: restored) + #expect(try await restored.readSymlink(path: "/dangling") == "/missing") + let workspace = Workspace(fileSystem: memory) + let checkpoint = try await workspace.createCheckpoint() + #expect(await workspace.exists("/dangling", at: .checkpoint(checkpoint.id))) + } + + @Test + func `mounted empty directories succeed and backend errors survive`() async throws { + let memory = InMemoryFileSystem() + try await memory.createDirectory(path: "/empty", recursive: false) + try await memory.writeFile(path: "/file", data: Data(), append: false) + let mounted = MountedFileSystem(base: memory) + #expect(try await mounted.listDirectory(path: "/empty").isEmpty) + for (path, code): (WorkspacePath, Int32) in [("/file", ENOTDIR), ("/missing", ENOENT)] { + do { + _ = try await mounted.listDirectory(path: path) + Issue.record("expected backend error") + } catch { + #expect((error as NSError).domain == NSPOSIXErrorDomain) + #expect((error as NSError).code == Int(code)) + } + } + let shadowedFile = MountedFileSystem(base: memory, mounts: [ + .init(mountPoint: "/file/inner", fileSystem: InMemoryFileSystem()) + ]) + #expect(try await shadowedFile.stat(path: "/file").kind == .file) + #expect(await shadowedFile.exists(path: "/file")) + let captured = try await Snapshot.capture(from: shadowedFile) + let restored = InMemoryFileSystem() + try await Snapshot.restore(captured, to: restored) + #expect(try await restored.stat(path: "/file").kind == .file) + do { + _ = try await shadowedFile.listDirectory(path: "/file") + Issue.record("expected ENOTDIR even with a nested mount") + } catch { #expect((error as NSError).code == Int(ENOTDIR)) } + let denied = MountedFileSystem(base: RegressionBackend(memory: memory, listError: EACCES), mounts: [ + .init(mountPoint: "/empty/inner", fileSystem: InMemoryFileSystem()), + .init(mountPoint: "/missing/inner", fileSystem: InMemoryFileSystem()), + ]) + for path: WorkspacePath in ["/empty", "/missing"] { + do { + _ = try await denied.listDirectory(path: path) + Issue.record("expected permission denial") + } catch { #expect((error as NSError).code == Int(EACCES)) } + } + } + + @Test + func `missing disk directories can supply synthetic mount parents`() async throws { + let root = try TestSupport.temporaryDirectory("SyntheticMountParent") + defer { TestSupport.remove(root) } + let disk = try LocalFileSystem(root: root) + let mounted = MountedFileSystem(mounts: [ + .init(mountPoint: "/", fileSystem: disk), + .init(mountPoint: "/parent/deep", fileSystem: InMemoryFileSystem()), + ]) + #expect(try await mounted.listDirectory(path: "/parent").map(\.name) == ["deep"]) + #expect(try await mounted.stat(path: "/parent").kind == .directory) + } + + @Test + func `nested mounts shadow entries and appear in glob and snapshot traversal`() async throws { + let root = InMemoryFileSystem() + try await root.writeFile(path: "/nested", data: Data("shadowed".utf8), append: false) + let nested = InMemoryFileSystem() + try await nested.writeFile(path: "/file", data: Data("visible".utf8), append: false) + let mounted = MountedFileSystem(mounts: [ + .init(mountPoint: "/", fileSystem: root), + .init(mountPoint: "/nested", fileSystem: nested), + .init(mountPoint: "/parent/deep", fileSystem: nested), + ]) + let entries = try await mounted.listDirectory(path: "/") + #expect(entries.map(\.name) == ["nested", "parent"]) + #expect(entries.allSatisfy { $0.info.kind == .directory }) + #expect(try await mounted.listDirectory(path: "/parent").map(\.name) == ["deep"]) + #expect(try await mounted.glob(pattern: "/**/file", currentDirectory: "/") == ["/nested/file", "/parent/deep/file"]) + let snapshot = try await Snapshot.capture(from: mounted) + let restored = InMemoryFileSystem() + try await Snapshot.restore(snapshot, to: restored) + #expect(try await restored.readFile(path: "/nested/file") == Data("visible".utf8)) + #expect(try await restored.readFile(path: "/parent/deep/file") == Data("visible".utf8)) + } +} + +// A real memory backend whose content-read boundary is unavailable, or whose directory +// listing is denied. This exercises metadata accounting and propagation, not a fake quota. +private final class RegressionBackend: FileSystem, Sendable { + let memory: InMemoryFileSystem + let denyReads: Bool + let listError: Int32? + let writeLatch: RegressionMutationLatch? + let denyChildStats: Bool + let omitIdentities: Bool + + init( + memory: InMemoryFileSystem, denyReads: Bool = false, + listError: Int32? = nil, writeLatch: RegressionMutationLatch? = nil, + denyChildStats: Bool = false, omitIdentities: Bool = false + ) { + self.memory = memory + self.denyReads = denyReads + self.listError = listError + self.writeLatch = writeLatch + self.denyChildStats = denyChildStats + self.omitIdentities = omitIdentities + } + func stat(path: WorkspacePath) async throws -> FileInfo { + if denyChildStats, !path.isRoot { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } + var info = try await memory.stat(path: path) + if omitIdentities { info.identity = nil } + return info + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + if let listError { throw NSError(domain: NSPOSIXErrorDomain, code: Int(listError)) } + return try await memory.listDirectory(path: path).map { entry in + var entry = entry + if omitIdentities { entry.info.identity = nil } + return entry + } + } + func readFile(path: WorkspacePath) async throws -> Data { + if denyReads { throw WorkspaceError.accessDenied(operation: "readFile", message: "content reads unavailable") } + return try await memory.readFile(path: path) + } + func exists(path: WorkspacePath) async -> Bool { await memory.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await memory.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + if path == "/first" { await writeLatch?.enter() } + try await memory.writeFile(path: path, data: data, append: append) + } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { + try await memory.createDirectory(path: path, recursive: recursive) + } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await memory.remove(path: path, recursive: recursive) } + func move(from: WorkspacePath, to: WorkspacePath) async throws { try await memory.move(from: from, to: to) } + func copy(from: WorkspacePath, to: WorkspacePath, recursive: Bool) async throws { + try await memory.copy(from: from, to: to, recursive: recursive) + } + func createSymlink(path: WorkspacePath, target: String) async throws { try await memory.createSymlink(path: path, target: target) } + func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { try await memory.createHardLink(path: path, target: target) } + func readSymlink(path: WorkspacePath) async throws -> String { try await memory.readSymlink(path: path) } +} + +private actor RegressionMutationLatch { + private var entered = false + private var released = false + private var observer: CheckedContinuation? + private var writer: CheckedContinuation? + + func enter() async { + entered = true + observer?.resume() + observer = nil + if !released { await withCheckedContinuation { writer = $0 } } + } + func waitUntilEntered() async { + if !entered { await withCheckedContinuation { observer = $0 } } + } + func release() { + released = true + writer?.resume() + writer = nil + } +} diff --git a/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift b/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift new file mode 100644 index 0000000..cda6b1d --- /dev/null +++ b/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift @@ -0,0 +1,170 @@ +#if os(Linux) +import Foundation +import Glibc +import Testing +@testable import Workspace + +@Suite("Linux recursive filesystem watching") +struct LinuxFileChangeStreamTests { + @Test + func `recursive watches observe writes inside hidden directories`() async throws { + let root = try TestSupport.temporaryDirectory("HiddenDirectoryWatch") + defer { TestSupport.remove(root) } + let hidden = root.appendingPathComponent(".config/nested", isDirectory: true) + try FileManager.default.createDirectory(at: hidden, withIntermediateDirectories: true) + let file = hidden.appendingPathComponent("file") + try Data("before".utf8).write(to: file) + let stream = try DirectoryChangeStream.observe(root, recursive: true) + let counter = LinuxInvalidationCounter() + let consumer = Task { for await _ in stream { await counter.increment() } } + defer { consumer.cancel() } + let before = await counter.value + try Data("after".utf8).write(to: file) + #expect(await counter.waitForChange(after: before)) + } + + @Test + func `removing thousands of watched directories promptly emits an invalidation`() async throws { + let root = try TestSupport.temporaryDirectory("BulkDirectoryRemoval") + defer { TestSupport.remove(root) } + let bulk = root.appendingPathComponent("bulk", isDirectory: true) + try FileManager.default.createDirectory(at: bulk, withIntermediateDirectories: true) + for index in 0..<6000 { + try FileManager.default.createDirectory( + atPath: bulk.path + "/d\(index)", withIntermediateDirectories: false + ) + } + let stream = try DirectoryChangeStream.observe(root, recursive: true) + let counter = LinuxInvalidationCounter() + let consumer = Task { for await _ in stream { await counter.increment() } } + defer { consumer.cancel() } + let clock = ContinuousClock() + let start = clock.now + try FileManager.default.removeItem(at: bulk) + #expect(await counter.waitForChange(after: 0)) + #expect(start.duration(to: clock.now) < .seconds(2)) + } + + @Test(.enabled(if: geteuid() != 0, "requires unprivileged filesystem permissions")) + func `an unreadable root parent does not disable observation`() async throws { + let parent = try TestSupport.temporaryDirectory("UnreadableWatchParent") + let root = parent.appendingPathComponent("root", isDirectory: true) + defer { _ = chmod(parent.path, 0o700); TestSupport.remove(parent) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let file = root.appendingPathComponent("file") + try Data("before".utf8).write(to: file) + // Execute-only access reproduces a non-owner under a 0711 parent. + #expect(chmod(parent.path, 0o111) == 0) + let stream = try DirectoryChangeStream.observe(root, recursive: true) + let counter = LinuxInvalidationCounter() + let consumer = Task { for await _ in stream { await counter.increment() } } + defer { consumer.cancel() } + try Data("after".utf8).write(to: file) + #expect(await counter.waitForChange(after: 0)) + } + + @Test(.enabled(if: geteuid() != 0, "requires unprivileged filesystem permissions")) + func `unreadable hidden children do not prevent initial or later watches`() async throws { + let root = try TestSupport.temporaryDirectory("UnreadableHiddenWatch") + let denied = root.appendingPathComponent(".denied", isDirectory: true) + defer { _ = chmod(denied.path, 0o700); TestSupport.remove(root) } + try FileManager.default.createDirectory(at: denied, withIntermediateDirectories: true) + let healthy = root.appendingPathComponent("healthy", isDirectory: true) + try FileManager.default.createDirectory(at: healthy, withIntermediateDirectories: true) + let file = healthy.appendingPathComponent("file") + try Data("before".utf8).write(to: file) + #expect(chmod(denied.path, 0) == 0) + let stream = try DirectoryChangeStream.observe(root, recursive: true) + let counter = LinuxInvalidationCounter() + let consumer = Task { for await _ in stream { await counter.increment() } } + defer { consumer.cancel() } + try Data("after".utf8).write(to: file) + #expect(await counter.waitForChange(after: 0)) + let beforeCreation = await counter.settledValue() + let later = root.appendingPathComponent("later", isDirectory: true) + try FileManager.default.createDirectory(at: later, withIntermediateDirectories: true) + let laterFile = later.appendingPathComponent("file") + try Data("before".utf8).write(to: laterFile) + #expect(await counter.waitForChange(after: beforeCreation)) + let settled = await counter.settledValue() + try Data("after".utf8).write(to: laterFile) + #expect(await counter.waitForChange(after: settled)) + } + + @Test + func `watch coverage resumes when the observed root is recreated later`() async throws { + let parent = try TestSupport.temporaryDirectory("ReplacementRootWatch") + defer { TestSupport.remove(parent) } + let root = parent.appendingPathComponent("root", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let stream = try DirectoryChangeStream.observe(root, recursive: true) + let counter = LinuxInvalidationCounter() + let consumer = Task { for await _ in stream { await counter.increment() } } + defer { consumer.cancel() } + let before = await counter.value + try FileManager.default.removeItem(at: root) + #expect(await counter.waitForChange(after: before)) + let beforeRecreation = await counter.settledValue() + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let file = root.appendingPathComponent("file") + try Data("new".utf8).write(to: file) + #expect(await counter.waitForChange(after: beforeRecreation)) + let settled = await counter.settledValue() + try Data("later".utf8).write(to: file) + #expect(await counter.waitForChange(after: settled)) + } + + @Test(arguments: [false, true]) + func `recursive watches resume after a directory inode is replaced`(move: Bool) async throws { + let root = try TestSupport.temporaryDirectory("ReplacementDirectoryWatch") + defer { TestSupport.remove(root) } + let watched = root.appendingPathComponent("watched", isDirectory: true) + try FileManager.default.createDirectory(at: watched, withIntermediateDirectories: true) + try Data("old".utf8).write(to: watched.appendingPathComponent("file")) + let stream = try DirectoryChangeStream.observe(root, recursive: true) + let counter = LinuxInvalidationCounter() + let consumer = Task { for await _ in stream { await counter.increment() } } + defer { consumer.cancel() } + let before = await counter.value + if move { + try FileManager.default.moveItem(at: watched, to: root.appendingPathComponent("moved")) + } else { + try FileManager.default.removeItem(at: watched) + } + // Recreate before the observer drains its events: existence at the old pathname + // must not keep the descriptor attached to the old inode. + try FileManager.default.createDirectory(at: watched, withIntermediateDirectories: true) + let replacement = watched.appendingPathComponent("file") + try Data("new".utf8).write(to: replacement) + #expect(await counter.waitForChange(after: before)) + let settled = await counter.settledValue() + // Only the child's watch sees this write; directory creation events cannot satisfy it. + try Data("later".utf8).write(to: replacement) + #expect(await counter.waitForChange(after: settled)) + } +} + +private actor LinuxInvalidationCounter { + private(set) var value = 0 + func increment() { value += 1 } + + func waitForChange(after previous: Int) async -> Bool { + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: .seconds(2)) + while clock.now < deadline { + if value > previous { return true } + try? await Task.sleep(for: .milliseconds(10)) + } + return value > previous + } + + func settledValue() async -> Int { + for _ in 0..<20 { + let previous = value + try? await Task.sleep(for: .milliseconds(100)) + if value == previous { return value } + } + return value + } +} +#endif From ec1875f1d2ecfc8cd62534e61b72aa94a138a4aa Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:44:08 -0700 Subject: [PATCH 08/34] Make three-way transaction merges structurally safe A transaction that removes or replaces a directory while the parent changes something beneath it (or the reverse) now reports a single .structural conflict at the outermost replaced directory instead of silently dropping the descendant. Descendant deletions converge with an ancestor's removal, identical changes coalesce, and every merged node is checked for a surviving directory ancestry before anything is written. Root-directory metadata participates in conflict detection and merging, so root permission changes are no longer silently dropped. Transactions enter a .committing state before the first await: edits, discard and a second commit throw TransactionError.committing while reads of the draft still work, and a rollback failure permanently closes the transaction. Snapshot.restore applies a directory's final permissions after its children, granting temporary owner access only when the filesystem actually denies it and restoring the original mode on failure, so restrictive directory modes no longer block their own contents. --- README.md | 9 + Sources/Workspace/Snapshot.swift | 104 +++- Sources/Workspace/Workspace+Merge.swift | 76 ++- Sources/Workspace/WorkspaceTransaction.swift | 170 +++-- Tests/WorkspaceTests/EditAtomicityTests.swift | 101 +-- Tests/WorkspaceTests/SnapshotTests.swift | 22 + Tests/WorkspaceTests/TestSupport.swift | 81 +++ Tests/WorkspaceTests/TransactionTests.swift | 585 ++++++++++++++++++ 8 files changed, 965 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 8500cfd..216797b 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,15 @@ if !commit.applied { } ``` +Transaction commits follow these rules: + +- Three-way merging accepts a change made by only one side, coalesces identical changes, and reports divergent changes to the same node as conflicts. +- A directory removed or replaced on one side while the other creates or modifies something beneath it is a `.structural` conflict at the outermost removed or replaced directory. Descendant deletions converge with ancestor removal. Directory mode changes and child edits merge. +- Strict mode rejects any parent change, including root-directory metadata. +- Moves are delete-plus-create operations. Moving the same source to different destinations keeps both copies. +- During `commit()`, edits and `discard()` throw `TransactionError.committing`; read-only draft calls remain available. Another commit also throws `.committing`. +- A rollback failure permanently closes the transaction. Failures that leave the parent unchanged allow retry. + Manual conflicted transactions remain active. The scoped convenience commits on success and discards on failure or conflict: ```swift diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index d5682f0..84428d7 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -153,6 +153,26 @@ extension Snapshot { return false } + static func isPermissionDenied(_ error: any Error) -> Bool { + if let posix = error as? POSIXError, posix.code == .EACCES || posix.code == .EPERM { + return true + } + let error = error as NSError + if error.domain == NSPOSIXErrorDomain, + error.code == POSIXError.Code.EACCES.rawValue || error.code == POSIXError.Code.EPERM.rawValue { + return true + } + if error.domain == NSCocoaErrorDomain, + error.code == CocoaError.fileReadNoPermission.rawValue || error.code == CocoaError.fileWriteNoPermission.rawValue { + return true + } + // corelibs Foundation may store a Swift Error rather than an NSError. + if let underlying = error.userInfo[NSUnderlyingErrorKey] as? any Error { + return isPermissionDenied(underlying) + } + return false + } + private static func captureEntry(from target: any FileSystem, at path: WorkspacePath) async throws -> Entry { guard let info = try await infoIfPresent(at: path, on: target) else { return .missing(.init(path: path)) @@ -197,7 +217,7 @@ extension Snapshot { if info.kind == .file { rewriteRequired = try await target.readFile(path: file.path) != file.data } else { - try await target.remove(path: file.path, recursive: true) + try await removeRecursively(at: file.path, on: target) rewriteRequired = true } } else { @@ -216,7 +236,7 @@ extension Snapshot { if info.kind == .symlink, try await target.readSymlink(path: symlink.path) == symlink.target { rewriteRequired = false } else { - try await target.remove(path: symlink.path, recursive: true) + try await removeRecursively(at: symlink.path, on: target) } } @@ -224,9 +244,10 @@ extension Snapshot { try await target.createSymlink(path: symlink.path, target: symlink.target) } case let .directory(directory): - try await ensureDirectory(at: directory.path, permissions: directory.permissions, on: target) - try await syncChildren(directory.children, under: directory.path, on: target) - try await setPermissionsIfNeeded(directory.permissions, at: directory.path, on: target) + try await ensureDirectory(at: directory.path, on: target) + try await syncDirectoryChildren( + directory.children, under: directory.path, finalPermissions: directory.permissions, on: target + ) } } @@ -253,7 +274,6 @@ extension Snapshot { private static func ensureDirectory( at path: WorkspacePath, - permissions: POSIXPermissions, on target: any FileSystem ) async throws { if let info = try await infoIfPresent(at: path, on: target) { @@ -268,6 +288,46 @@ extension Snapshot { } } + /// Most backends need no mode adjustment. Retry only a real permission denial, + /// and preserve the original mode on failure after granting temporary access. + private static func syncDirectoryChildren( + _ children: [Entry], + under path: WorkspacePath, + finalPermissions: POSIXPermissions? = nil, + on target: any FileSystem + ) async throws { + do { + try await syncChildren(children, under: path, on: target) + } catch let deniedError { + guard isPermissionDenied(deniedError) else { throw deniedError } + let original = try await target.stat(path: path).permissions + let working = POSIXPermissions(rawValue: original.rawValue | 0o700) + guard working != original else { throw deniedError } + do { + try await target.setPermissions(path: path, permissions: working) + } catch { + // The original denial explains the failure better than a refused grant. + throw deniedError + } + do { + // Synchronization is idempotent even if the first attempt made progress. + try await syncChildren(children, under: path, on: target) + try await setPermissionsIfNeeded(finalPermissions ?? original, at: path, on: target) + } catch { + do { + try await setPermissionsIfNeeded(original, at: path, on: target) + } catch let cleanupError { + throw WorkspaceRollbackError(originalError: error, rollbackError: cleanupError) + } + throw error + } + return + } + if let finalPermissions { + try await setPermissionsIfNeeded(finalPermissions, at: path, on: target) + } + } + private static func syncChildren( _ expectedChildren: [Entry], under parentPath: WorkspacePath, @@ -277,7 +337,7 @@ extension Snapshot { let existingEntries = try await target.listDirectory(path: parentPath) for entry in existingEntries where !expectedNames.contains(entry.name) { - try await target.remove(path: parentPath.appending(entry.name), recursive: true) + try await removeRecursively(at: parentPath.appending(entry.name), on: target) } for child in expectedChildren.sorted(by: { $0.path < $1.path }) { @@ -285,24 +345,26 @@ extension Snapshot { } } - private static func removeEntryIfPresent(at path: WorkspacePath, on target: any FileSystem) async throws { - guard let info = try await infoIfPresent(at: path, on: target) else { return } - - if path.isRoot { - let existingEntries = try await target.listDirectory(path: path) - for entry in existingEntries { - try await target.remove(path: path.appending(entry.name), recursive: true) - } - return + private static func removeRecursively(at path: WorkspacePath, on target: any FileSystem) async throws { + do { + try await target.remove(path: path, recursive: true) + } catch { + guard isPermissionDenied(error) else { throw error } + // The failure may come from a restrictive descendant, rather than + // this directory. Each directory grants access only after its own + // child synchronization fails, and restores its mode before removal. + try await removeEntryIfPresent(at: path, on: target) } + } + private static func removeEntryIfPresent(at path: WorkspacePath, on target: any FileSystem) async throws { + guard let info = try await infoIfPresent(at: path, on: target) else { return } if info.kind == .directory { - let existingEntries = try await target.listDirectory(path: path) - for entry in existingEntries { - try await target.remove(path: path.appending(entry.name), recursive: true) - } + try await syncDirectoryChildren([], under: path, on: target) + } + if !path.isRoot { + try await target.remove(path: path, recursive: true) } - try await target.remove(path: path, recursive: true) } } diff --git a/Sources/Workspace/Workspace+Merge.swift b/Sources/Workspace/Workspace+Merge.swift index fb1d13d..9e0c7cf 100644 --- a/Sources/Workspace/Workspace+Merge.swift +++ b/Sources/Workspace/Workspace+Merge.swift @@ -29,46 +29,66 @@ extension Workspace { } } - static func buildEntryTree( - from nodes: [WorkspacePath: MergeNode], - rootPermissions: POSIXPermissions - ) -> Snapshot.Entry { - var childNames: [WorkspacePath: [String]] = [:] - for path in nodes.keys { - childNames[path.parent, default: []].append(path.name) + /// Report only the outermost removed/replaced directory for each surviving + /// descendant change. Descendant deletions converge with ancestor removal. + static func structuralMergeConflicts( + base: [WorkspacePath: MergeNode], + parent: [WorkspacePath: MergeNode], + draft: [WorkspacePath: MergeNode] + ) -> Set { + var conflicts: Set = [] + let paths = Set(base.keys).union(parent.keys).union(draft.keys) + for path in paths where parent[path] != base[path] || draft[path] != base[path] { + var ancestor = path + var outermost: WorkspacePath? + while !ancestor.isRoot { + ancestor = ancestor.parent + let parentReplaced = base[ancestor]?.kind == .directory + && parent[ancestor]?.kind != .directory && draft[ancestor]?.kind == .directory + let draftReplaced = base[ancestor]?.kind == .directory + && draft[ancestor]?.kind != .directory && parent[ancestor]?.kind == .directory + if (parentReplaced && draft[path] != nil && draft[path] != base[path]) + || (draftReplaced && parent[path] != nil && parent[path] != base[path]) { + outermost = ancestor + } + } + if let outermost { conflicts.insert(outermost) } } + return conflicts + } - func subtree(at path: WorkspacePath, node: MergeNode?) -> Snapshot.Entry { - switch node?.kind ?? .directory { + static func buildEntryTree(from nodes: [WorkspacePath: MergeNode]) throws -> Snapshot.Entry { + guard let root = nodes[.root], root.kind == .directory else { + throw WorkspaceError.mutationFailed("merged tree must have a directory root") + } + var childNames: [WorkspacePath: [(String, MergeNode)]] = [:] + for (path, node) in nodes where !path.isRoot { + // Checking every immediate parent proves the full ancestry inductively, + // since WorkspacePath parents strictly shorten toward the validated root. + guard nodes[path.parent]?.kind == .directory else { + throw WorkspaceError.mutationFailed("merged node \(path) has no directory parent") + } + childNames[path.parent, default: []].append((path.name, node)) + } + + func subtree(at path: WorkspacePath, node: MergeNode) -> Snapshot.Entry { + switch node.kind { case .file: - return .file( - .init( - path: path, - data: node?.fileData ?? Data(), - permissions: node?.permissions ?? .defaultFile - ) - ) + return .file(.init(path: path, data: node.fileData ?? Data(), permissions: node.permissions)) case .symlink: - return .symlink( - .init( - path: path, - target: node?.symlinkTarget ?? "", - permissions: node?.permissions ?? POSIXPermissions(0o777) - ) - ) + return .symlink(.init(path: path, target: node.symlinkTarget ?? "", permissions: node.permissions)) case .directory: return .directory( .init( path: path, - permissions: node?.permissions ?? .defaultDirectory, - children: (childNames[path] ?? []).sorted().map { name in - let child = path.appending(name) - return subtree(at: child, node: nodes[child]) + permissions: node.permissions, + children: (childNames[path] ?? []).sorted { $0.0 < $1.0 }.map { name, child in + subtree(at: path.appending(name), node: child) } ) ) } } - return subtree(at: .root, node: MergeNode(kind: .directory, permissions: rootPermissions)) + return subtree(at: .root, node: root) } } diff --git a/Sources/Workspace/WorkspaceTransaction.swift b/Sources/Workspace/WorkspaceTransaction.swift index 9341f4d..83def33 100644 --- a/Sources/Workspace/WorkspaceTransaction.swift +++ b/Sources/Workspace/WorkspaceTransaction.swift @@ -4,7 +4,10 @@ extension Workspace { /// An isolated, long-lived editing session with preview, commit, and discard. public actor Transaction { public enum CommitStrategy: String, Sendable, Codable { + /// Reject any parent change since the transaction began, including root metadata. case strict + /// Merge whole nodes: identical changes converge; divergent changes conflict. + /// Ancestor replacements also conflict with incompatible descendant changes. case threeWay } @@ -14,6 +17,9 @@ extension Workspace { case modifiedAndDeleted case bothCreated case parentChanged + /// The conflict's path is the directory removed or replaced on one + /// side while the other side changed something beneath it. + case structural } public var path: WorkspacePath @@ -57,10 +63,12 @@ extension Workspace { public enum TransactionError: Swift.Error, Sendable { case inactive + /// The transaction is committing and cannot accept edits, discard, or another commit. + case committing case conflicts([Conflict]) } - enum State { case active, committed, discarded } + enum State { case active, committing, committed, discarded, failed } public nonisolated let id: UUID private let parent: Workspace @@ -69,6 +77,8 @@ extension Workspace { private let baseCheckpointID: UUID? private let label: String? private var state: State = .active + private var pendingEdits = 0 + private var editsDrained: CheckedContinuation? init(parent: Workspace, draft: Workspace, base: Snapshot, baseCheckpointID: UUID?, label: String?) { id = UUID() @@ -80,12 +90,12 @@ extension Workspace { } public func readData(from path: WorkspacePath) async throws -> Data { - try requireActive() + try requireReadable() return try await draft.readData(from: path) } public func readText(_ path: WorkspacePath) async throws -> String { - try requireActive() + try requireReadable() return try await draft.readText(path) } @@ -93,72 +103,77 @@ extension Workspace { _ type: T.Type = T.self, from path: WorkspacePath ) async throws -> T { - try requireActive() + try requireReadable() return try await draft.readJSON(type, from: path) } public func exists(_ path: WorkspacePath) async throws -> Bool { - try requireActive() + try requireReadable() return await draft.exists(path) } public func info(_ path: WorkspacePath) async throws -> FileInfo { - try requireActive() + try requireReadable() return try await draft.info(path) } public func list(_ path: WorkspacePath) async throws -> [DirectoryEntry] { - try requireActive() + try requireReadable() return try await draft.list(path) } public func glob(_ pattern: String, currentDirectory: WorkspacePath = .root) async throws -> [WorkspacePath] { - try requireActive() + try requireReadable() return try await draft.glob(pattern, currentDirectory: currentDirectory) } public func tree(at path: WorkspacePath = .root, maxDepth: Int? = nil) async throws -> FileTree { - try requireActive() + try requireReadable() return try await draft.tree(at: path, maxDepth: maxDepth) } public func search(_ request: SearchRequest) async throws -> SearchResult { - try requireActive() + try requireReadable() return try await draft.search(request) } public func preview(_ edits: [Edit]) async throws -> ChangeSet { - try requireActive() + try requireReadable() return try await draft.preview(edits) } @discardableResult public func apply(_ edits: [Edit], policy: EditPolicy = .atomic) async throws -> EditResult { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.apply(edits, policy: policy) } @discardableResult public func writeText(_ path: WorkspacePath, _ content: String) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.writeText(path, content) } @discardableResult public func appendText(_ path: WorkspacePath, _ content: String) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.appendText(path, content) } @discardableResult public func writeData(_ data: Data, to path: WorkspacePath) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.writeData(data, to: path) } @discardableResult public func appendData(_ path: WorkspacePath, _ data: Data) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.appendData(path, data) } @@ -168,19 +183,22 @@ extension Workspace { to path: WorkspacePath, prettyPrinted: Bool = true ) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.writeJSON(value, to: path, prettyPrinted: prettyPrinted) } @discardableResult public func createDirectory(_ path: WorkspacePath, recursive: Bool = true) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.createDirectory(path, recursive: recursive) } @discardableResult public func remove(_ path: WorkspacePath, recursive: Bool = true) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.remove(path, recursive: recursive) } @@ -188,56 +206,75 @@ extension Workspace { public func copy(from source: WorkspacePath, to destination: WorkspacePath, recursive: Bool = true) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.copy(from: source, to: destination, recursive: recursive) } @discardableResult public func move(from source: WorkspacePath, to destination: WorkspacePath) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.move(from: source, to: destination) } @discardableResult public func createSymbolicLink(_ path: WorkspacePath, target: String) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.createSymbolicLink(path, target: target) } @discardableResult public func createHardLink(_ path: WorkspacePath, target: WorkspacePath) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.createHardLink(path, target: target) } @discardableResult public func setPermissions(_ permissions: POSIXPermissions, at path: WorkspacePath) async throws -> ChangeSet { - try requireActive() + try beginEdit() + defer { endEdit() } return try await draft.setPermissions(permissions, at: path) } /// Returns all base-to-draft changes without mutating the parent workspace. public func preview() async throws -> ChangeSet { - try requireActive() + try requireReadable() let current = try await draft.revisionSnapshot(.current) return ChangeSet.compare(before: base, after: current, maxTextBytes: DiffOptions.default.maxTextBytes) } - /// Attempts to commit this transaction. Conflicted transactions remain active. + /// Attempts to commit this transaction. Conflicts and safely rolled-back failures + /// leave it active for retry. A rollback failure permanently closes it. public func commit(strategy: CommitStrategy = .threeWay) async throws -> Commit { try requireActive() - let draftSnapshot = try await draft.revisionSnapshot(.current) - let result = try await parent.commitTransaction( - base: base, - draft: draftSnapshot, - baseCheckpointID: baseCheckpointID, - label: label, - strategy: strategy - ) - if result.conflicts.isEmpty { - state = .committed + state = .committing + do { + // An edit admitted before commit may still be suspended in the draft. + // Drain those edits before capture; new edits are rejected by requireActive. + if pendingEdits > 0 { + await withCheckedContinuation { editsDrained = $0 } + } + try Task.checkCancellation() + let draftSnapshot = try await draft.revisionSnapshot(.current) + let result = try await parent.commitTransaction( + base: base, + draft: draftSnapshot, + baseCheckpointID: baseCheckpointID, + label: label, + strategy: strategy + ) + state = result.conflicts.isEmpty ? .committed : .active + return result + } catch { + // The parent's gate covers restoration too. An ordinary error means + // no mutation occurred or compensation succeeded. Failed compensation + // may have left filesystem or history changes behind: never allow retry. + state = error is WorkspaceRollbackError ? .failed : .active + throw error } - return result } /// Permanently discards the transaction. @@ -246,7 +283,25 @@ extension Workspace { state = .discarded } + private func beginEdit() throws { + try requireActive() + pendingEdits += 1 + } + + private func endEdit() { + pendingEdits -= 1 + if pendingEdits == 0 { + editsDrained?.resume() + editsDrained = nil + } + } + + private func requireReadable() throws { + guard state == .active || state == .committing else { throw TransactionError.inactive } + } + private func requireActive() throws { + if state == .committing { throw TransactionError.committing } guard state == .active else { throw TransactionError.inactive } } } @@ -321,9 +376,6 @@ extension Workspace { Self.flattenMergeNodes(base.entry, into: &baseNodes) Self.flattenMergeNodes(current.entry, into: ¤tNodes) Self.flattenMergeNodes(draft.entry, into: &draftNodes) - baseNodes.removeValue(forKey: .root) - currentNodes.removeValue(forKey: .root) - draftNodes.removeValue(forKey: .root) if strategy == .strict, currentNodes != baseNodes { let changed = Set(baseNodes.keys).union(currentNodes.keys).filter { baseNodes[$0] != currentNodes[$0] } @@ -336,8 +388,14 @@ extension Workspace { } var merged = currentNodes - var conflicts: [Transaction.Conflict] = [] + let structuralPaths = Self.structuralMergeConflicts(base: baseNodes, parent: currentNodes, draft: draftNodes) + var conflicts = structuralPaths.sorted().map { Transaction.Conflict(path: $0, kind: .structural) } for path in Set(baseNodes.keys).union(currentNodes.keys).union(draftNodes.keys).sorted() { + // One structural conflict describes the entire affected subtree; + // suppress redundant node conflicts beneath it. + var ancestor = path + while !ancestor.isRoot && !structuralPaths.contains(ancestor) { ancestor = ancestor.parent } + if structuralPaths.contains(ancestor) { continue } let baseNode = baseNodes[path] let parentNode = currentNodes[path] let transactionNode = draftNodes[path] @@ -368,27 +426,35 @@ extension Workspace { preview: preview, appliedChanges: ChangeSet(), checkpoint: nil, - conflicts: conflicts + conflicts: conflicts.sorted { $0.path < $1.path } ) } - let rootPermissions: POSIXPermissions = if case let .directory(directory) = current.entry { - directory.permissions - } else { - .defaultDirectory - } - let mergedSnapshot = Snapshot( - rootPath: .root, - entry: Self.buildEntryTree(from: merged, rootPermissions: rootPermissions) - ) + // Reconstruction validates the whole flat map before the first parent write. + let mergedSnapshot = Snapshot(rootPath: .root, entry: try Self.buildEntryTree(from: merged)) let applied = ChangeSet.compare(before: current, after: mergedSnapshot, maxTextBytes: DiffOptions.default.maxTextBytes) guard !applied.isEmpty else { return Transaction.Commit(preview: preview, appliedChanges: applied, checkpoint: nil, conflicts: []) } + // A child-only change may sit below a directory the parent made read-only. + // Include restrictive ancestors so restore can grant temporary owner access + // and reapply their final modes after restoring the children. The operation + // gate still owns the complete capture/restore/rollback sequence. + var restoreRoots = Set(applied.touchedPaths) + for path in applied.touchedPaths { + var ancestor = path + while !ancestor.isRoot { + ancestor = ancestor.parent + if let node = currentNodes[ancestor], node.kind == .directory, + node.permissions.rawValue & 0o700 != 0o700 { + restoreRoots.insert(ancestor) + } + } + } var before: [Snapshot] = [] var after: [Snapshot] = [] - for path in applied.touchedPaths { + for path in restoreRoots.sorted() { if before.contains(where: { Snapshot.contains($0.rootPath, path) }) { continue } let oldEntry = Self.snapshotEntry(at: path, in: current.entry) ?? .missing(.init(path: path)) let newEntry = Self.snapshotEntry(at: path, in: mergedSnapshot.entry) ?? .missing(.init(path: path)) diff --git a/Tests/WorkspaceTests/EditAtomicityTests.swift b/Tests/WorkspaceTests/EditAtomicityTests.swift index 7d8555b..8ddffe6 100644 --- a/Tests/WorkspaceTests/EditAtomicityTests.swift +++ b/Tests/WorkspaceTests/EditAtomicityTests.swift @@ -6,7 +6,7 @@ import Testing struct EditAtomicityTests { @Test func `single write does not read unrelated files`() async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() try await fs.base.writeFile(path: "/unrelated", data: Data("unrelated".utf8), append: false) let workspace = Workspace(fileSystem: fs) try await workspace.writeText("/changed", "new") @@ -26,10 +26,10 @@ struct EditAtomicityTests { @Test func `rollback works without optional permissions support`() async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() try await fs.base.writeFile(path: "/a", data: Data("old".utf8), append: false) let workspace = Workspace(fileSystem: fs) - await #expect(throws: EditTestError.self) { + await #expect(throws: TestFileSystemError.self) { try await workspace.apply([.writeText("/a", "new"), .writeText("/fail", "fail")]) } #expect(try await workspace.readText("/a") == "old") @@ -62,14 +62,14 @@ struct EditAtomicityTests { let root = try TestSupport.temporaryDirectory() defer { TestSupport.remove(root) } let local = try LocalFileSystem(root: root) - let fs = EditTestFileSystem(base: local) + let fs = HookableFileSystem(base: local) await fs.onWrite { path in if path == "/a" { try Data("external".utf8).write(to: root.appendingPathComponent("external")) } } let workspace = Workspace(fileSystem: fs) - await #expect(throws: EditTestError.self) { + await #expect(throws: TestFileSystemError.self) { try await workspace.apply([.writeText("/a", "new"), .writeText("/fail", "fail")]) } #expect(try await local.readFile(path: "/external") == Data("external".utf8)) @@ -82,7 +82,7 @@ struct EditAtomicityTests { defer { TestSupport.remove(root) } let storeURL = root.appendingPathComponent("file") try Data().write(to: storeURL) - let fs = EditTestFileSystem() + let fs = HookableFileSystem() let workspace = Workspace(fileSystem: fs, persistence: .directory(storeURL)) await #expect(throws: (any Error).self) { try await workspace.writeText("/a", "new") } #expect(await fs.writes.isEmpty) @@ -91,7 +91,7 @@ struct EditAtomicityTests { @Test func `wrapped access denial during capture is never treated as absence`() async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() try await fs.base.writeFile(path: "/a", data: Data("original".utf8), append: false) let denial = NSError( domain: NSCocoaErrorDomain, @@ -107,29 +107,29 @@ struct EditAtomicityTests { @Test func `post mutation capture failure rolls back`() async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() try await fs.base.writeFile(path: "/a", data: Data("old".utf8), append: false) await fs.onWrite { path in if path == "/a" { await fs.failNextRead() } } let workspace = Workspace(fileSystem: fs) - await #expect(throws: EditTestError.self) { try await workspace.writeText("/a", "new") } + await #expect(throws: TestFileSystemError.self) { try await workspace.writeText("/a", "new") } #expect(try await workspace.readText("/a") == "old") #expect(try await workspace.history().isEmpty) } @Test func `concurrent successful edit survives atomic rollback with consistent history and events`() async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() try await fs.base.writeFile(path: "/a", data: Data("original".utf8), append: false) let workspace = Workspace(fileSystem: fs) let events = await workspace.events() - let entered = EditSignal() - let resume = EditSignal() + let entered = TestSignal() + let resume = TestSignal() await fs.onWrite { _ in await entered.signal() await resume.wait() } let failing = Task { - await #expect(throws: EditTestError.self) { + await #expect(throws: TestFileSystemError.self) { try await workspace.apply([.writeText("/a", "temporary"), .writeText("/fail", "fail")]) } } @@ -228,7 +228,7 @@ struct EditAtomicityTests { @Test(arguments: [EditOperation.transaction, .checkpoint, .archive]) func `parent restoration serializes against concurrent apply`(operation: EditOperation) async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() let workspace = Workspace(fileSystem: fs) try await workspace.writeText("/a", "target") let checkpoint = try await workspace.createCheckpoint() @@ -237,8 +237,8 @@ struct EditAtomicityTests { let transaction = try await workspace.beginTransaction() try await transaction.writeText("/a", "target") let events = await workspace.events(.init(includeCheckpoints: false)) - let entered = EditSignal() - let resume = EditSignal() + let entered = TestSignal() + let resume = TestSignal() await fs.onWrite { _ in await entered.signal() await resume.wait() @@ -322,15 +322,15 @@ struct EditAtomicityTests { @Test func `rollback failure carries both errors`() async throws { - let fs = EditTestFileSystem() + let fs = HookableFileSystem() await fs.failRemovals() let workspace = Workspace(fileSystem: fs) do { try await workspace.apply([.writeText("/a", "new"), .writeText("/fail", "fail")]) Issue.record("expected rollback failure") } catch let error as WorkspaceRollbackError { - #expect(error.originalError is EditTestError) - #expect(error.rollbackError is EditTestError) + #expect(error.originalError is TestFileSystemError) + #expect(error.rollbackError is TestFileSystemError) #expect(error.description == "injected; restoring the previous state also failed: injected") } } @@ -340,69 +340,6 @@ enum EditOperation: CaseIterable, Sendable { case edit, archive, checkpoint, transaction } -private enum EditTestError: Error { case injected } - -private actor EditTestFileSystem: FileSystem { - let base: any FileSystem - var reads: [WorkspacePath] = [] - var writes: [WorkspacePath] = [] - private var writeHook: (@Sendable (WorkspacePath) async throws -> Void)? - private var readFailure = false - private var statError: NSError? - func failStat(_ error: NSError) { statError = error } - private var removeFailure = false - func failRemovals() { removeFailure = true } - - init(base: any FileSystem = InMemoryFileSystem()) { self.base = base } - func onWrite(_ hook: @escaping @Sendable (WorkspacePath) async throws -> Void) { writeHook = hook } - func failNextRead() { readFailure = true } - func stat(path: WorkspacePath) async throws -> FileInfo { - if let statError { throw statError } - return try await base.stat(path: path) - } - func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } - func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } - func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { - try await base.glob(pattern: pattern, currentDirectory: currentDirectory) - } - func readFile(path: WorkspacePath) async throws -> Data { - reads.append(path) - if readFailure { readFailure = false; throw EditTestError.injected } - return try await base.readFile(path: path) - } - func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { - writes.append(path) - let hook = writeHook - writeHook = nil - try await hook?(path) - if path == "/fail" { throw EditTestError.injected } - try await base.writeFile(path: path, data: data, append: append) - } - func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } - func remove(path: WorkspacePath, recursive: Bool) async throws { - if removeFailure { throw EditTestError.injected } - try await base.remove(path: path, recursive: recursive) - } - func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } - func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { - try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) - } -} - -private actor EditSignal { - private var signaled = false - private var waiters: [CheckedContinuation] = [] - func wait() async { - if signaled { return } - await withCheckedContinuation { waiters.append($0) } - } - func signal() { - signaled = true - for waiter in waiters { waiter.resume() } - waiters.removeAll() - } -} - func nextEditEvent( in stream: AsyncStream, timeout: Duration = .seconds(2) ) async -> WorkspaceEvent? { diff --git a/Tests/WorkspaceTests/SnapshotTests.swift b/Tests/WorkspaceTests/SnapshotTests.swift index 5deeee1..6696504 100644 --- a/Tests/WorkspaceTests/SnapshotTests.swift +++ b/Tests/WorkspaceTests/SnapshotTests.swift @@ -4,6 +4,28 @@ import Testing @Suite("Snapshot engine") struct SnapshotTests { + @Test + func `permission denial recognizes Foundation and nested Swift error representations`() { + let denied: [any Error] = [ + POSIXError(.EACCES), POSIXError(.EPERM), + NSError(domain: NSPOSIXErrorDomain, code: Int(POSIXError.Code.EACCES.rawValue)), + NSError(domain: NSPOSIXErrorDomain, code: Int(POSIXError.Code.EPERM.rawValue)), + CocoaError(.fileReadNoPermission), CocoaError(.fileWriteNoPermission), + NSError(domain: NSCocoaErrorDomain, code: CocoaError.fileWriteUnknown.rawValue, + userInfo: [NSUnderlyingErrorKey: POSIXError(.EACCES) as any Error]), + NSError(domain: "outer", code: 1, userInfo: [NSUnderlyingErrorKey: + NSError(domain: "inner", code: 2, userInfo: [NSUnderlyingErrorKey: POSIXError(.EPERM) as any Error])]) + ] + for error in denied { #expect(Snapshot.isPermissionDenied(error)) } + let other: [any Error] = [ + POSIXError(.ENOENT), POSIXError(.EIO), CocoaError(.fileWriteUnknown), + WorkspaceError.unsupported("permissions"), WorkspaceError.accessDenied(operation: "write", message: nil), + NSError(domain: "unrelated", code: Int(POSIXError.Code.EACCES.rawValue)), + NSError(domain: "outer", code: 1, userInfo: [NSUnderlyingErrorKey: POSIXError(.EIO) as any Error]) + ] + for error in other { #expect(!Snapshot.isPermissionDenied(error)) } + } + @Test func `folding a replacement preserves unrelated missing entries`() { let missing = Snapshot.Entry.missing(.init(path: "/unrelated")) diff --git a/Tests/WorkspaceTests/TestSupport.swift b/Tests/WorkspaceTests/TestSupport.swift index d13af3d..c6afc24 100644 --- a/Tests/WorkspaceTests/TestSupport.swift +++ b/Tests/WorkspaceTests/TestSupport.swift @@ -1,4 +1,5 @@ import Foundation +@testable import Workspace enum TestSupport { static func temporaryDirectory(_ prefix: String = "WorkspaceTests") throws -> URL { @@ -12,3 +13,83 @@ enum TestSupport { try? FileManager.default.removeItem(at: url) } } + +enum TestFileSystemError: Error { case injected } + +actor HookableFileSystem: FileSystem { + let base: any FileSystem + var reads: [WorkspacePath] = [] + var writes: [WorkspacePath] = [] + var permissionWrites: [(path: WorkspacePath, permissions: POSIXPermissions)] = [] + private let forwardsPermissions: Bool + private var writeHook: (@Sendable (WorkspacePath) async throws -> Void)? + private var readHook: (@Sendable (WorkspacePath) async throws -> Void)? + private var readFailure = false + private var statError: NSError? + func failStat(_ error: NSError) { statError = error } + private var removeFailure = false + func failRemovals() { removeFailure = true } + + init(base: any FileSystem = InMemoryFileSystem(), forwardsPermissions: Bool = false) { + self.base = base + self.forwardsPermissions = forwardsPermissions + } + func onWrite(_ hook: @escaping @Sendable (WorkspacePath) async throws -> Void) { writeHook = hook } + func onRead(_ hook: @escaping @Sendable (WorkspacePath) async throws -> Void) { readHook = hook } + func failNextRead() { readFailure = true } + func stat(path: WorkspacePath) async throws -> FileInfo { + if let statError { throw statError } + return try await base.stat(path: path) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await base.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func readFile(path: WorkspacePath) async throws -> Data { + reads.append(path) + let hook = readHook + readHook = nil + try await hook?(path) + if readFailure { readFailure = false; throw TestFileSystemError.injected } + return try await base.readFile(path: path) + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + writes.append(path) + let hook = writeHook + writeHook = nil + try await hook?(path) + if path == "/fail" { throw TestFileSystemError.injected } + try await base.writeFile(path: path, data: data, append: append) + } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { + if removeFailure { throw TestFileSystemError.injected } + try await base.remove(path: path, recursive: recursive) + } + func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { + permissionWrites.append((path, permissions)) + guard forwardsPermissions else { + throw WorkspaceError.unsupported("setting permissions is not supported by this filesystem") + } + try await base.setPermissions(path: path, permissions: permissions) + } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) + } +} + +actor TestSignal { + private var signaled = false + private var waiters: [CheckedContinuation] = [] + func wait() async { + if signaled { return } + await withCheckedContinuation { waiters.append($0) } + } + func signal() { + signaled = true + for waiter in waiters { waiter.resume() } + waiters.removeAll() + } +} diff --git a/Tests/WorkspaceTests/TransactionTests.swift b/Tests/WorkspaceTests/TransactionTests.swift index 0bf8234..8a22bad 100644 --- a/Tests/WorkspaceTests/TransactionTests.swift +++ b/Tests/WorkspaceTests/TransactionTests.swift @@ -1,5 +1,10 @@ import Foundation import Testing +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif @testable import Workspace @Suite("Transactions") @@ -91,4 +96,584 @@ struct TransactionTests { #expect(result.commit.applied) #expect(try await workspace.readText("/value") == "42") } + + enum NodeShape: String, CaseIterable, Sendable { + case missing, file, directory, symlink + + var create: [Edit] { + switch self { + case .missing: [] + case .file: [.writeText("/d", "file")] + case .directory: [.createDirectory("/d")] + case .symlink: [.createSymbolicLink("/d", target: "target")] + } + } + + func replacing(_ base: NodeShape) -> [Edit] { + if self == base { return [] } + return (base == .missing ? [] : [.remove("/d")]) + create + } + } + + // All ordered type combinations, including identical replacements and + // unilateral changes. Expected outcomes are explicit tree shapes. + @Test(arguments: NodeShape.allCases, NodeShape.allCases) + func `type replacements merge identically or conflict without losing nodes`( + base: NodeShape, replacement: NodeShape + ) async throws { + for parentShape in NodeShape.allCases { + let workspace = Workspace() + try await workspace.apply(base.create) + let transaction = try await workspace.beginTransaction() + try await transaction.apply(replacement.replacing(base)) + try await workspace.apply(parentShape.replacing(base)) + let before = try await workspace.revisionSnapshot(.current).entry + let history = try await workspace.history() + let commit = try await transaction.commit() + let shouldConflict = replacement != base && parentShape != base && replacement != parentShape + #expect(commit.applied == !shouldConflict, + "base=\(base), draft=\(replacement), parent=\(parentShape)") + if shouldConflict { + let expectedKind: Workspace.Transaction.Conflict.Kind = if base == .missing { + .bothCreated + } else if parentShape == .missing || replacement == .missing { + .modifiedAndDeleted + } else { + .bothModified + } + #expect(commit.conflicts.map(\.path) == ["/d"]) + #expect(commit.conflicts.map(\.kind) == [expectedKind]) + #expect(try await workspace.revisionSnapshot(.current).entry == before) + #expect(try await workspace.history() == history) + #expect(commit.checkpoint == nil) + } else { + let expected = Workspace() + try await expected.apply((replacement == base ? parentShape : replacement).create) + #expect(try await workspace.revisionSnapshot(.current).entry == expected.revisionSnapshot(.current).entry) + } + } + } + + enum DescendantChange: String, CaseIterable, Sendable { + case create, modify, permissions, typeChange, remove, move, moveOut + var base: [Edit] { + self == .create ? [.createDirectory("/d")] : [.createDirectory("/d/nested"), .writeText("/d/nested/a", "base")] + } + var edits: [Edit] { + switch self { + case .create: [.writeText("/d/new.txt", "new")] + case .modify: [.writeText("/d/nested/a", "changed")] + case .permissions: [.setPermissions("/d/nested/a", POSIXPermissions(0o600))] + case .typeChange: [.remove("/d/nested/a"), .createDirectory("/d/nested/a")] + case .remove: [.remove("/d/nested/a")] + case .moveOut: [.move(from: "/d/nested/a", to: "/out")] + case .move: [.move(from: "/d/nested/a", to: "/d/moved")] + } + } + } + + @Test(arguments: [NodeShape.missing, .file, .symlink], DescendantChange.allCases) + func `ancestor replacement and descendant edits never silently lose work`( + replacement: NodeShape, descendant: DescendantChange + ) async throws { + for reverse in [false, true] { + let workspace = Workspace() + try await workspace.apply(descendant.base) + let transaction = try await workspace.beginTransaction() + let ancestorEdits = replacement.replacing(.directory) + try await transaction.apply(reverse ? descendant.edits : ancestorEdits) + try await workspace.apply(reverse ? ancestorEdits : descendant.edits) + let before = try await workspace.revisionSnapshot(.current).entry + let history = try await workspace.history() + let commit = try await transaction.commit() + if descendant == .remove || descendant == .moveOut { + #expect(commit.applied) + let expected = Workspace() + try await expected.apply(replacement.create) + if descendant == .moveOut { try await expected.writeText("/out", "base") } + #expect(try await workspace.revisionSnapshot(.current).entry == expected.revisionSnapshot(.current).entry) + } else { + // In particular, modifying /d/nested/a reports only the outer /d, + // without nested structural or modifiedAndDeleted duplicates. + #expect(commit.conflicts == [.init(path: "/d", kind: .structural)]) + #expect(try await workspace.revisionSnapshot(.current).entry == before) + #expect(try await workspace.history() == history) + #expect(commit.checkpoint == nil) + try await transaction.discard() + } + } + } + + @Test(arguments: [WorkspacePath.root, "/d", "/d/a"]) + func `permissions use three-way rules including root metadata`(path: WorkspacePath) async throws { + // nil leaves that side unchanged; equal and differing edits are separate cases. + let pairs: [(Int?, Int?, Bool)] = [ + (0o700, nil, true), (nil, 0o700, true), + (0o700, 0o700, true), (0o700, 0o711, false) + ] + for (draftMode, parentMode, clean) in pairs { + let workspace = Workspace() + try await workspace.createDirectory("/d") + try await workspace.writeText("/d/a", "content") + let transaction = try await workspace.beginTransaction() + if let draftMode { try await transaction.setPermissions(POSIXPermissions(draftMode), at: path) } + if let parentMode { try await workspace.setPermissions(POSIXPermissions(parentMode), at: path) } + let before = try await workspace.revisionSnapshot(.current).entry + let commit = try await transaction.commit() + #expect(commit.applied == clean) + if clean { + #expect(try await workspace.info(path).permissions == POSIXPermissions(draftMode ?? parentMode!)) + #expect(try await workspace.readText("/d/a") == "content") + if draftMode != nil && parentMode == nil { + #expect(commit.appliedChanges.touchedPaths == [path]) + #expect(commit.checkpoint != nil) + } + } else { + #expect(commit.conflicts.map(\.path) == [path]) + #expect(try await workspace.revisionSnapshot(.current).entry == before) + } + } + } + + @Test + func `strict commits include root permissions in comparison and application`() async throws { + let workspace = Workspace() + let transaction = try await workspace.beginTransaction() + try await workspace.setPermissions(POSIXPermissions(0o700), at: .root) + let rejected = try await transaction.commit(strategy: .strict) + #expect(rejected.conflicts == [.init(path: .root, kind: .parentChanged)]) + try await workspace.setPermissions(.defaultDirectory, at: .root) + try await transaction.setPermissions(POSIXPermissions(0o711), at: .root) + #expect(try await transaction.commit(strategy: .strict).applied) + #expect(try await workspace.info(.root).permissions == POSIXPermissions(0o711)) + } + + @Test + func `commit excludes late edits discard and duplicate commits while suspended`() async throws { + let fs = HookableFileSystem() + let workspace = Workspace(fileSystem: fs) + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "42") + let entered = TestSignal() + let resume = TestSignal() + await fs.onWrite { _ in + await entered.signal() + await resume.wait() + } + let committing = Task { try await transaction.commit() } + await entered.wait() + await #expect { try await transaction.writeText("/late", "lost") } throws: { error in + guard case Workspace.Transaction.TransactionError.committing = error else { return false } + return true + } + await #expect { try await transaction.discard() } throws: { error in + guard case Workspace.Transaction.TransactionError.committing = error else { return false } + return true + } + await #expect { + try await transaction.commit() + } throws: { error in + guard case Workspace.Transaction.TransactionError.committing = error else { return false } + return true + } + #expect(try await transaction.exists("/a")) + #expect(try await transaction.readText("/a") == "42") + #expect(try await transaction.readData(from: "/a") == Data("42".utf8)) + #expect(try await transaction.readJSON(Int.self, from: "/a") == 42) + #expect(try await transaction.search(.init(pattern: .literal("42"))).matches.map(\.path) == ["/a"]) + #expect(try await transaction.info("/a").kind == .file) + #expect(try await transaction.list(.root).map(\.name) == ["a"]) + #expect(try await transaction.tree().children?.map(\.path) == ["/a"]) + #expect(try await transaction.glob("/a*") == ["/a"]) + #expect(try await transaction.preview().touchedPaths == ["/a"]) + #expect(try await transaction.preview([.writeText("/preview", "scratch")]).touchedPaths == ["/preview"]) + #expect(!(try await transaction.exists("/preview"))) + await resume.signal() + #expect(try await committing.value.applied) + #expect(try await workspace.readText("/a") == "42") + #expect(!(await workspace.exists("/late"))) + await #expect { try await transaction.commit() } throws: { error in + guard case Workspace.Transaction.TransactionError.inactive = error else { return false } + return true + } + } + + struct MergePair: Sendable { + let name: String + let base: [Edit] + let draft: [Edit] + let parent: [Edit] + // nil means conflict; otherwise these edits create the complete expected tree. + let expected: [Edit]? + var conflict: Workspace.Transaction.Conflict? = nil + + static let cases: [MergePair] = [ + .init(name: "identical file edits", base: [.writeText("/a", "old")], + draft: [.writeText("/a", "same")], parent: [.writeText("/a", "same")], + expected: [.writeText("/a", "same")]), + .init(name: "different file edits", base: [.writeText("/a", "old")], + draft: [.writeText("/a", "draft")], parent: [.writeText("/a", "parent")], expected: nil, + conflict: .init(path: "/a", kind: .bothModified)), + .init(name: "identical link targets", base: [.createSymbolicLink("/a", target: "old")], + draft: [.remove("/a"), .createSymbolicLink("/a", target: "same")], + parent: [.remove("/a"), .createSymbolicLink("/a", target: "same")], + expected: [.createSymbolicLink("/a", target: "same")]), + .init(name: "different link targets", base: [.createSymbolicLink("/a", target: "old")], + draft: [.remove("/a"), .createSymbolicLink("/a", target: "draft")], + parent: [.remove("/a"), .createSymbolicLink("/a", target: "parent")], expected: nil, + conflict: .init(path: "/a", kind: .bothModified)), + .init(name: "content versus mode is a node conflict", base: [.writeText("/a", "old")], + draft: [.writeText("/a", "new")], parent: [.setPermissions("/a", POSIXPermissions(0o600))], expected: nil, + conflict: .init(path: "/a", kind: .bothModified)), + .init(name: "directory mode and child content merge", base: [.createDirectory("/d"), .writeText("/d/a", "old")], + draft: [.writeText("/d/a", "new")], parent: [.setPermissions("/d", POSIXPermissions(0o700))], + expected: [.createDirectory("/d"), .writeText("/d/a", "new"), .setPermissions("/d", POSIXPermissions(0o700))]), + .init(name: "concurrent directory creation keeps both children", base: [], + draft: [.createDirectory("/d"), .writeText("/d/a", "a")], + parent: [.createDirectory("/d"), .writeText("/d/b", "b")], + expected: [.createDirectory("/d"), .writeText("/d/a", "a"), .writeText("/d/b", "b")]), + .init(name: "identical populated directory deletions", base: [.createDirectory("/d"), .writeText("/d/a", "a")], + draft: [.remove("/d")], parent: [.remove("/d")], expected: []), + .init(name: "move directory versus new source child", base: [.createDirectory("/d")], + draft: [.move(from: "/d", to: "/moved")], parent: [.writeText("/d/new", "new")], expected: nil, + conflict: .init(path: "/d", kind: .structural)), + .init(name: "move file versus source edit", base: [.writeText("/a", "old")], + draft: [.move(from: "/a", to: "/moved")], parent: [.writeText("/a", "new")], expected: nil, + conflict: .init(path: "/a", kind: .modifiedAndDeleted)), + .init(name: "move child outside a removed ancestor", base: [.createDirectory("/d"), .writeText("/d/a", "a")], + draft: [.move(from: "/d/a", to: "/out")], parent: [.remove("/d")], + expected: [.writeText("/out", "a")]), + .init(name: "identical directory moves", base: [.createDirectory("/d"), .writeText("/d/a", "a")], + draft: [.move(from: "/d", to: "/moved")], parent: [.move(from: "/d", to: "/moved")], + expected: [.createDirectory("/moved"), .writeText("/moved/a", "a")]), + .init(name: "different move destinations preserve both creations", base: [.writeText("/a", "a")], + draft: [.move(from: "/a", to: "/one")], parent: [.move(from: "/a", to: "/two")], + expected: [.writeText("/one", "a"), .writeText("/two", "a")]) + ] + } + + @Test(arguments: MergePair.cases) + func `concurrent changes merge to the expected complete tree or leave parent intact`(pair: MergePair) async throws { + for reverse in [false, true] { + let workspace = Workspace() + try await workspace.apply(pair.base) + let transaction = try await workspace.beginTransaction() + try await transaction.apply(reverse ? pair.parent : pair.draft) + try await workspace.apply(reverse ? pair.draft : pair.parent) + let before = try await workspace.revisionSnapshot(.current).entry + let history = try await workspace.history() + let checkpoints = try await workspace.checkpoints() + let commit = try await transaction.commit() + if let expectedEdits = pair.expected { + #expect(commit.applied, "\(pair.name)") + let expected = Workspace() + try await expected.apply(expectedEdits) + #expect(try await workspace.revisionSnapshot(.current).entry == expected.revisionSnapshot(.current).entry) + } else { + let conflict = try #require(pair.conflict) + #expect(commit.conflicts.map(\.path) == [conflict.path], "\(pair.name)") + #expect(commit.conflicts.map(\.kind) == [conflict.kind], "\(pair.name)") + #expect(try await workspace.revisionSnapshot(.current).entry == before) + #expect(try await workspace.history() == history) + #expect(try await workspace.checkpoints() == checkpoints) + } + } + } + + @Test(arguments: [false, true]) + func `commit can retry after capture failure or successful rollback`(duringWrite: Bool) async throws { + let fs = HookableFileSystem() + let workspace = Workspace(fileSystem: fs) + try await workspace.writeText("/a", "base") + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "draft") + let before = try await workspace.revisionSnapshot(.current).entry + let history = try await workspace.history() + if duringWrite { + await fs.onWrite { _ in + // Fail after touching the parent, so this exercises compensation. + try await fs.base.writeFile(path: "/a", data: Data("partial".utf8), append: false) + throw TestFileSystemError.injected + } + } else { + await fs.onRead { _ in throw TestFileSystemError.injected } + } + await #expect(throws: TestFileSystemError.self) { try await transaction.commit() } + #expect(try await workspace.revisionSnapshot(.current).entry == before) + #expect(try await workspace.history() == history) + #expect(try await workspace.checkpoints().isEmpty) + try await transaction.appendText("/a", "-retry") + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText("/a") == "draft-retry") + } + + @Test + func `failed rollback closes transaction against unsafe retries`() async throws { + let fs = HookableFileSystem() + let workspace = Workspace(fileSystem: fs) + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "draft") + await fs.onWrite { _ in + try await fs.base.writeFile(path: "/a", data: Data("partial".utf8), append: false) + await fs.failRemovals() + throw TestFileSystemError.injected + } + await #expect(throws: WorkspaceRollbackError.self) { try await transaction.commit() } + #expect(try await workspace.readText("/a") == "partial") + await #expect { try await transaction.writeText("/a", "retry") } throws: { error in + guard case Workspace.Transaction.TransactionError.inactive = error else { return false } + return true + } + await #expect { try await transaction.commit() } throws: { error in + guard case Workspace.Transaction.TransactionError.inactive = error else { return false } + return true + } + await #expect { try await transaction.discard() } throws: { error in + guard case Workspace.Transaction.TransactionError.inactive = error else { return false } + return true + } + } + + @Test + func `tree reconstruction rejects missing or non-directory ancestry`() throws { + let root = Workspace.MergeNode(kind: .directory, permissions: .defaultDirectory) + let file = Workspace.MergeNode(kind: .file, permissions: .defaultFile, fileData: Data("keep".utf8)) + let link = Workspace.MergeNode(kind: .symlink, permissions: POSIXPermissions(0o777), symlinkTarget: "elsewhere") + let invalid: [[WorkspacePath: Workspace.MergeNode]] = [ + [:], [.root: file], [.root: link], + [.root: root, "/missing/child": file], + [.root: root, "/d": file, "/d/child": file], + [.root: root, "/d": link, "/d/child": file], + [.root: root, "/missing/deep": root, "/missing/deep/child": file] + ] + for nodes in invalid { + #expect(throws: WorkspaceError.self) { try Workspace.buildEntryTree(from: nodes) } + } + } + + @Test + func `commit waits for an already admitted draft edit before taking its snapshot`() async throws { + let workspace = Workspace() + let base = try await workspace.revisionSnapshot(.current) + let draftFS = HookableFileSystem() + let draft = Workspace(fileSystem: draftFS) + // Use the existing initializer to supply a draft whose I/O can suspend. + let transaction = Workspace.Transaction(parent: workspace, draft: draft, base: base, baseCheckpointID: nil, label: nil) + let entered = TestSignal() + let resume = TestSignal() + await draftFS.onWrite { _ in + await entered.signal() + await resume.wait() + } + let editing = Task { try await transaction.apply([.writeText("/a", "first"), .writeText("/b", "second")]) } + await entered.wait() + let committing = Task { + try await commitWithSuspensionHook(transaction) { _ in + let entries = try await workspace.list(.root) + #expect(entries.isEmpty) + await resume.signal() + } + } + _ = try await editing.value + #expect(try await committing.value.applied) + #expect(try await workspace.readText("/a") == "first") + #expect(try await workspace.readText("/b") == "second") + } + + @Test + func `cancelled commit before parent mutation remains retryable`() async throws { + let workspace = Workspace() + let base = try await workspace.revisionSnapshot(.current) + let draftFS = HookableFileSystem() + let draft = Workspace(fileSystem: draftFS) + let transaction = Workspace.Transaction(parent: workspace, draft: draft, base: base, baseCheckpointID: nil, label: nil) + try await transaction.writeText("/a", "draft") + let entered = TestSignal() + let resume = TestSignal() + await draftFS.onRead { _ in + await entered.signal() + await resume.wait() + } + await workspace.acquireOperation() + let committing = Task { try await transaction.commit() } + await entered.wait() + committing.cancel() + await resume.signal() + await workspace.releaseOperation() + await #expect(throws: CancellationError.self) { try await committing.value } + #expect(try await workspace.list(.root).isEmpty) + #expect(try await workspace.history().isEmpty) + try await transaction.appendText("/a", "-retry") + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText("/a") == "draft-retry") + } + + @Test(arguments: [false, true]) + func `restrictive in-memory directories need no chmod for commit or rollback`(atomicFailure: Bool) async throws { + let fs = HookableFileSystem() + try await fs.base.createDirectory(path: "/d", recursive: true) + try await fs.base.writeFile(path: "/d/a", data: Data("old".utf8), append: false) + try await fs.base.setPermissions(path: "/d", permissions: POSIXPermissions(0o555)) + let workspace = Workspace(fileSystem: fs) + try await workspace.writeText("/d/plain", "plain") + if atomicFailure { + // The no-op mkdir expands the rollback journal to /d, whose mode + // already matches the snapshot and needs no optional chmod support. + await #expect(throws: TestFileSystemError.self) { + try await workspace.apply([ + .writeText("/d/a", "temporary"), .createDirectory("/d"), .writeText("/fail", "fail") + ]) + } + #expect(try await workspace.readText("/d/a") == "old") + } else { + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/d/a", "committed") + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText("/d/a") == "committed") + } + #expect(try await workspace.readText("/d/plain") == "plain") + #expect(try await workspace.info("/d").permissions == POSIXPermissions(0o555)) + #expect(await fs.permissionWrites.isEmpty) + } + + @Test + func `unchanged restrictive subdirectories are not chmodded during restore`() async throws { + let fs = HookableFileSystem(forwardsPermissions: true) + try await fs.base.createDirectory(path: "/d/kept", recursive: true) + try await fs.base.writeFile(path: "/d/kept/a", data: Data("kept".utf8), append: false) + try await fs.base.setPermissions(path: "/d/kept", permissions: POSIXPermissions(0o555)) + let workspace = Workspace(fileSystem: fs) + let transaction = try await workspace.beginTransaction() + try await transaction.setPermissions(POSIXPermissions(0o750), at: "/d") + #expect(try await transaction.commit().applied) + // Only the explicitly requested parent mode change reaches the backend. + #expect(await fs.permissionWrites.map(\.path) == ["/d"]) + #expect(await fs.permissionWrites.map(\.permissions) == [POSIXPermissions(0o750)]) + #expect(try await workspace.info("/d/kept").permissions == POSIXPermissions(0o555)) + #expect(try await workspace.readText("/d/kept/a") == "kept") + } + + @Test(.enabled(if: geteuid() != 0, "Root bypasses directory write permissions")) + func `local child failure restores temporary access and leaves transaction retryable`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: root.appendingPathComponent("d").path) + TestSupport.remove(root) + } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/d", recursive: true) + try await local.writeFile(path: "/d/a", data: Data("old".utf8), append: false) + try await local.setPermissions(path: "/d", permissions: POSIXPermissions(0o500)) + let fs = HookableFileSystem(base: local, forwardsPermissions: true) + let workspace = Workspace(fileSystem: fs) + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/d/a", "new") + await fs.onWrite { _ in + if try await local.stat(path: "/d").permissions == POSIXPermissions(0o500) { + // Let the actual local write fail with EACCES first. The retry + // after granting access must fail with the injected error. + await fs.onWrite { _ in throw TestFileSystemError.injected } + } else { + throw TestFileSystemError.injected + } + } + await #expect(throws: TestFileSystemError.self) { try await transaction.commit() } + #expect(try await workspace.info("/d").permissions == POSIXPermissions(0o500)) + #expect(try await workspace.readText("/d/a") == "old") + #expect(await fs.permissionWrites.map(\.permissions) == [POSIXPermissions(0o700), POSIXPermissions(0o500)]) + #expect(try await workspace.history().isEmpty) + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText("/d/a") == "new") + #expect(try await workspace.info("/d").permissions == POSIXPermissions(0o500)) + } + + enum RestrictiveRemoval: CaseIterable, Sendable { + case directory, child, replaceWithFile + } + + @Test(.enabled(if: geteuid() != 0, "Root bypasses directory write permissions"), + arguments: RestrictiveRemoval.allCases) + func `local restore removes directories with restrictive descendants`(scenario: RestrictiveRemoval) async throws { + let root = try TestSupport.temporaryDirectory() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: root.appendingPathComponent("d").path) + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: root.appendingPathComponent("d/nested").path) + TestSupport.remove(root) + } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/d/nested", recursive: true) + try await local.writeFile(path: "/d/nested/a", data: Data("remove".utf8), append: false) + try await local.setPermissions(path: "/d/nested", permissions: POSIXPermissions(0o500)) + try await local.setPermissions(path: "/d", permissions: POSIXPermissions(0o500)) + let workspace = Workspace(fileSystem: local) + let transaction = try await workspace.beginTransaction() + try await transaction.remove(scenario == .child ? "/d/nested" : "/d") + if scenario == .replaceWithFile { try await transaction.writeText("/d", "replacement") } + #expect(try await transaction.commit().applied) + switch scenario { + case .directory: #expect(!(await workspace.exists("/d"))) + case .child: + #expect(try await workspace.list("/d").isEmpty) + #expect(try await workspace.info("/d").permissions == POSIXPermissions(0o500)) + case .replaceWithFile: #expect(try await workspace.readText("/d") == "replacement") + } + } + + enum RestrictiveDirectory: CaseIterable, Sendable { + case parentMode, draftMode, existingMode, newDirectory + } + + @Test(.enabled(if: geteuid() != 0, "Root bypasses directory write permissions"), + arguments: RestrictiveDirectory.allCases) + func `local commits restore children before restrictive directory modes`(scenario: RestrictiveDirectory) async throws { + let root = try TestSupport.temporaryDirectory() + defer { + // Restore traversal/write permissions before cleaning up, including + // when an assertion or commit fails. + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: root.appendingPathComponent("d").path) + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: root.appendingPathComponent("d/nested").path) + TestSupport.remove(root) + } + let workspace = Workspace(fileSystem: try LocalFileSystem(root: root)) + if scenario != .newDirectory { + try await workspace.createDirectory("/d") + try await workspace.writeText("/d/kept", "kept") + } + if scenario == .existingMode { try await workspace.setPermissions(POSIXPermissions(0o500), at: "/d") } + let transaction = try await workspace.beginTransaction() + if scenario == .parentMode { try await workspace.setPermissions(POSIXPermissions(0o500), at: "/d") } + if scenario == .draftMode { + try await workspace.createDirectory("/d/nested") + try await workspace.writeText("/d/nested/new", "new") + } else { + try await transaction.createDirectory("/d/nested") + try await transaction.writeText("/d/nested/new", "new") + } + if scenario == .draftMode || scenario == .newDirectory { + try await transaction.setPermissions(POSIXPermissions(0o500), at: "/d") + } + // A new restrictive child directory must also be populated before chmod. + if scenario == .newDirectory { try await transaction.setPermissions(POSIXPermissions(0o500), at: "/d/nested") } + #expect(try await transaction.commit().applied) + #expect(try await workspace.info("/d").permissions == POSIXPermissions(0o500)) + #expect(try await workspace.readText("/d/nested/new") == "new") + if scenario == .newDirectory { + #expect(try await workspace.info("/d/nested").permissions == POSIXPermissions(0o500)) + } else { + #expect(try await workspace.readText("/d/kept") == "kept") + } + } +} + +// Enqueue the hook on the same actor before entering commit synchronously. It +// cannot run until commit suspends, so a paused draft write is released only +// after commit has entered its drain. This adds no production observation seam. +private func commitWithSuspensionHook( + _ transaction: isolated Workspace.Transaction, + hook: @escaping @Sendable (isolated Workspace.Transaction) async throws -> Void +) async throws -> Workspace.Transaction.Commit { + let observer = Task { try await hook(transaction) } + let result = try await transaction.commit() + try await observer.value + return result } From 9e08b68401db20c656a1e00bf194c52864f57b14 Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 20:47:04 -0700 Subject: [PATCH 09/34] Make filesystem accounting exact for local disks and dangling links LocalFileSystem reports file identity (device, inode) and link count for non-directories, so LimitedFileSystem charges hard-link aliases exactly on disk instead of falling back to a conservative bound that could reject valid appends. Local advertises replacementFollowsFinalSymlink: replacement writes follow in-root final links and atomically replace the target, while links that escape the root are rejected. LimitedFileSystem resolves a dangling final link through the backend's own physical resolution (a new internal potential-path protocol forwarded by Mounted, Sandbox and SecurityScoped), so relative targets reached through symlinked parents and absolute in-root host paths are charged where the write actually lands. InMemoryFileSystem writes resolve links the same way its reads do. OverlayFileSystem and AuthorizedFileSystem now give exists (and exact-path glob) lstat semantics like the other backends, so dangling symlinks survive snapshots and checkpoints through an overlay. --- Sources/Workspace/FS/FileSystem.swift | 6 + Sources/Workspace/FS/InMemoryFileSystem.swift | 40 +-- Sources/Workspace/FS/LimitedFileSystem.swift | 23 +- Sources/Workspace/FS/LocalFileSystem.swift | 40 ++- Sources/Workspace/FS/MountedFileSystem.swift | 2 +- Sources/Workspace/FS/OverlayFileSystem.swift | 5 +- Sources/Workspace/FS/SandboxFileSystem.swift | 6 +- .../FS/SecurityScopedFileSystem.swift | 6 +- Sources/Workspace/Support/Permissions.swift | 10 +- .../FileSystemFollowupTests.swift | 273 ++++++++++++++++++ .../FileSystemRegressionTests.swift | 13 +- 11 files changed, 368 insertions(+), 56 deletions(-) create mode 100644 Tests/WorkspaceTests/FileSystemFollowupTests.swift diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index 93a76a3..ae4d060 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -214,6 +214,12 @@ public protocol FileSystemChangeSource: FileSystem { func changes() async throws -> AsyncStream } +/// Resolves links in the backend's namespace without requiring the target to exist. +/// Missing parent suffixes are retained for quota preflight without creating them. +protocol PotentialPathResolvingFileSystem: FileSystem { + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath +} + extension FileSystem { /// The default implementation throws ``WorkspaceError/unsupported(_:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index d89e1b1..c62f94d 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -13,7 +13,7 @@ import Glibc /// /// Actor isolation serializes access to the tree so the filesystem can be shared safely across /// concurrent tasks. -public actor InMemoryFileSystem: FileSystem { +public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { private final class Node { enum Kind { case file(Data) @@ -134,15 +134,7 @@ public actor InMemoryFileSystem: FileSystem { throw posixError(EISDIR) } - var writePath = path - var depth = 0 - while depth < 64, let symlinkTarget = try symlinkTargetIfPresent(at: writePath) { - writePath = WorkspacePath(normalizing: symlinkTarget, relativeTo: writePath.dirname) - depth += 1 - } - if depth >= 64 { - throw posixError(ELOOP) - } + let writePath = try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0, allowMissing: true) let (parent, name) = try parentDirectoryAndName(for: writePath) var children = try directoryChildren(of: parent) @@ -480,18 +472,6 @@ public actor InMemoryFileSystem: FileSystem { return (parent, name, entry) } - private func symlinkTargetIfPresent(at path: WorkspacePath) throws -> String? { - guard let (_, _, entry) = try parentDirectoryEntryIfPresent(for: path) else { - return nil - } - - if case let .symlink(target) = entry.kind { - return target - } - - return nil - } - private func node(at path: WorkspacePath, followFinalSymlink: Bool, symlinkDepth: Int = 0) throws -> Node { if symlinkDepth > 64 { throw posixError(ELOOP) @@ -531,7 +511,13 @@ public actor InMemoryFileSystem: FileSystem { return current } - private func resolvePath(path: WorkspacePath, followFinalSymlink: Bool, symlinkDepth: Int) throws -> WorkspacePath { + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0, allowMissing: true) + } + + private func resolvePath( + path: WorkspacePath, followFinalSymlink: Bool, symlinkDepth: Int, allowMissing: Bool = false + ) throws -> WorkspacePath { if symlinkDepth > 64 { throw posixError(ELOOP) } @@ -550,6 +536,9 @@ public actor InMemoryFileSystem: FileSystem { } guard let child = children[component] else { + if allowMissing { + return resolvedPath.appending(components.dropFirst(index).joined(separator: "/")) + } throw posixError(ENOENT) } @@ -559,7 +548,10 @@ public actor InMemoryFileSystem: FileSystem { let targetPath = WorkspacePath(normalizing: target, relativeTo: resolvedPath) let remaining = components.suffix(from: index + 1).joined(separator: "/") let combined = remaining.isEmpty ? targetPath : targetPath.appending(remaining) - return try resolvePath(path: combined, followFinalSymlink: followFinalSymlink, symlinkDepth: symlinkDepth + 1) + return try resolvePath( + path: combined, followFinalSymlink: followFinalSymlink, + symlinkDepth: symlinkDepth + 1, allowMissing: allowMissing + ) } current = child diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index 1b71cda..9f283a3 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -46,7 +46,7 @@ public enum FileSystemLimitError: Error, Sendable, Equatable, CustomStringConver /// Custom backends must follow standard mutation semantics: replacement changes one entry; /// append updates aliases; parents may be created. A composition cycle (for example a Mounted /// base that mounts this same LimitedFileSystem) deadlocks while the gate is held. -public actor LimitedFileSystem: FileSystem { +public actor LimitedFileSystem: FileSystem, PotentialPathResolvingFileSystem { private let base: any FileSystem private let pathResolver: FileSystemPathResolver public let limits: FileSystemLimits @@ -75,6 +75,9 @@ public actor LimitedFileSystem: FileSystem { public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { try await base.resolveRealPath(path: path) } + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + try await pathResolver.resolve(path, features: await base.capabilities(), followFinal: true) + } public func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } public func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { try await base.glob(pattern: pattern, currentDirectory: currentDirectory) @@ -333,17 +336,13 @@ public actor LimitedFileSystem: FileSystem { } } -// Resolves in the backend's own namespace. A Mounted backend owns the interpretation of -// absolute symlink targets; only it can translate a dangling target back to a virtual path. -// Only a directly held MountedFileSystem is special-cased. When another decorator wraps -// MountedFileSystem, dangling links inside a mount fall back to virtual-namespace resolution. +// Prefer backend-owned resolution so physical parents and absolute targets retain their +// backend meaning, including through decorators. Older backends use the heuristic below. struct FileSystemPathResolver: Sendable { private let base: any FileSystem - private let mounted: MountedFileSystem? init(base: any FileSystem) { self.base = base - mounted = base as? MountedFileSystem } func resolve( @@ -355,6 +354,9 @@ struct FileSystemPathResolver: Sendable { return try await resolve(path.dirname, in: tree, features: features, followFinal: true, depth: depth) .appending(path.basename) } + if let resolving = base as? any PotentialPathResolvingFileSystem { + return try await resolving.resolvePotentialPath(path: path) + } if features.contains(.realPathResolution) { var ancestor = path var missing: [String] = [] @@ -368,9 +370,11 @@ struct FileSystemPathResolver: Sendable { // uses lstat paths. Ask the backend to resolve those intermediate links. let info = try? await base.stat(path: ancestor) if info?.kind == .symlink { - if let mounted { return try await mounted.resolvePotentialPath(path: path) } let target = try await base.readSymlink(path: ancestor) - let targetPath = try WorkspacePath(validating: target, relativeTo: ancestor.dirname) + let parent = try await resolve( + ancestor.dirname, in: tree, features: features, followFinal: true, depth: depth + 1 + ) + let targetPath = try WorkspacePath(validating: target, relativeTo: parent) let combined = missing.isEmpty ? targetPath : targetPath.appending(missing.reversed().joined(separator: "/")) return try await resolve(combined, in: tree, features: features, followFinal: true, depth: depth + 1) } @@ -379,7 +383,6 @@ struct FileSystemPathResolver: Sendable { } } } - if let mounted { return try await mounted.resolvePotentialPath(path: path) } var current = WorkspacePath.root for (index, component) in path.components.enumerated() { current = current.appending(component) diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 7e92d92..f2f9b3b 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -18,7 +18,7 @@ private typealias LocalStat = Glibc.stat /// This is a safety layer, not a sandbox: a host process can relocate an already-open directory /// outside the root, change hard-linked data, or mutate entries during recursive operations. /// Watch subscriptions still use the configured URL and are not descriptor-pinned. -public final class LocalFileSystem: FileSystem, @unchecked Sendable { +public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// Returns zero on success or a captured POSIX errno on failure. Injection is per-instance /// so unsupported filesystem behavior can be tested without mounts or global syscall hooks. typealias ExclusiveRename = @Sendable (Int32, String, Int32, String) -> Int32 @@ -169,6 +169,7 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { private func resolve( _ path: WorkspacePath, followFinal: Bool, createParents: Bool = false, + allowMissingAncestors: Bool = false, configuration: ConfigurationSnapshot? = nil ) throws -> Entry { let config = try configuration ?? requireConfiguration() @@ -178,6 +179,7 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { var pending = path.components var current = config.root var names: [String] = [] + var missingAncestor = false var hops = 0 while !pending.isEmpty { let name = pending.removeFirst() @@ -190,9 +192,16 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { } names.removeLast() current = config.root + missingAncestor = false for ancestor in names { let entry = Entry(parent: current, name: ancestor, path: .root) - current = try openEntry(entry, flags: Self.searchDirectoryFlags) + do { + current = try openEntry(entry, flags: Self.searchDirectoryFlags) + } catch let error as NSError where allowMissingAncestors + && error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { + missingAncestor = true + break + } } continue } @@ -203,6 +212,13 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { if final && !followFinal { return entry } + // Only potential-path resolution permits missing ancestors. The descriptor is + // no longer an operation target; this mode consumes only the resulting path. + if missingAncestor { + if final { return entry } + names.append(name) + continue + } var info: LocalStat do { info = try metadata(entry) @@ -210,6 +226,11 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { if final { return entry } + if allowMissingAncestors { + names.append(name) + missingAncestor = true + continue + } guard createParents else { throw error } @@ -260,7 +281,9 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { return FileInfo( path: path, kind: type == S_IFLNK ? .symlink : (type == S_IFDIR ? .directory : .file), size: UInt64(max(0, info.st_size)), permissions: POSIXPermissions(Int(info.st_mode & 0o7777)), - modificationDate: Date(timeIntervalSince1970: Double(modified.tv_sec) + Double(modified.tv_nsec) / 1e9)) + modificationDate: Date(timeIntervalSince1970: Double(modified.tv_sec) + Double(modified.tv_nsec) / 1e9), + identity: FileIdentity(device: UInt64(truncatingIfNeeded: info.st_dev), inode: UInt64(info.st_ino)), + linkCount: type == S_IFDIR ? nil : UInt64(info.st_nlink)) } /// See ``FileSystem/stat(path:)``. @@ -415,8 +438,9 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { /// See ``FileSystem/writeFile(path:data:append:)``. /// - /// Replacement is atomic and breaks hard-link sharing; append preserves the opened inode. - /// Symlink targets are resolved explicitly before either operation. + /// Replacement writes follow in-root final symlinks and atomically replace the target, + /// breaking its hard-link sharing. Links that escape the root are rejected. + /// Appends preserve the opened inode. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { let entry = try resolve(path, followFinal: true, createParents: true) if append { @@ -650,6 +674,10 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { return entry.path } + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + try resolve(path, followFinal: true, allowMissingAncestors: true).path + } + /// See ``FileSystem/exists(path:)``. /// /// Uses lstat semantics, so a final dangling symlink still exists. @@ -659,7 +687,7 @@ public final class LocalFileSystem: FileSystem, @unchecked Sendable { /// See ``FileSystem/capabilities()``. public func capabilities() async -> FileSystemFeatures { - [.symlinks, .hardLinks, .permissions, .realPathResolution] + [.symlinks, .hardLinks, .permissions, .realPathResolution, .replacementFollowsFinalSymlink] } /// See ``FileSystem/glob(pattern:currentDirectory:)``. diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index c2adeab..3ccf068 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -10,7 +10,7 @@ import Glibc /// /// Each mount is exposed at a virtual path, allowing callers to present a single workspace assembled /// from multiple independent roots. -public final class MountedFileSystem: FileSystem, @unchecked Sendable { +public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// A mounted filesystem and the virtual path where it appears. public struct Mount: Sendable { /// The normalized virtual mount point. diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index 2d9aee6..17dd44e 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -14,6 +14,9 @@ import Glibc /// copied up by a mutation. Deletions are recorded as whiteouts that hide the lower entry, and /// directory listings merge both layers with upper entries shadowing lower ones. /// +/// File identities are forwarded from whichever layer holds the entry. Copy-up gives a file a new +/// identity, so limits charge appends to a hard-linked lower file conservatively. +/// /// `reload()` (or `configure(rootDirectory:)`) discards the upper layer and whiteouts, so the /// overlay reflects the source directory's current state again. public actor OverlayFileSystem: FileSystem { @@ -183,7 +186,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/exists(path:)``. public func exists(path: WorkspacePath) async -> Bool { - guard let target = try? await resolveMergedPath(path) else { + guard let target = try? await resolveMergedPath(path, followFinal: false) else { return false } return await mergedEntry(target) != nil diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index 4ec9d82..b23825b 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -1,7 +1,7 @@ import Foundation /// A disk-backed filesystem rooted in a standard app sandbox location. -public final class SandboxFileSystem: FileSystem, Sendable { +public final class SandboxFileSystem: FileSystem, PotentialPathResolvingFileSystem, Sendable { /// Supported strategies for choosing the sandbox root. public enum Root: Sendable { /// The user's documents directory. @@ -63,6 +63,10 @@ public final class SandboxFileSystem: FileSystem, Sendable { await backing.capabilities() } + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + try await backing.resolvePotentialPath(path: path) + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await backing.writeFile(path: path, data: data, append: append) diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index 18c306c..d6f1810 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -5,7 +5,7 @@ import Foundation /// /// This adapter is intended for iOS and macOS workflows where filesystem access must be granted through /// a bookmark or document picker URL. -public final class SecurityScopedFileSystem: FileSystem, @unchecked Sendable { +public final class SecurityScopedFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// The permitted mutability for the scoped filesystem. public enum AccessMode: Sendable { /// Only read operations are allowed. @@ -167,6 +167,10 @@ public final class SecurityScopedFileSystem: FileSystem, @unchecked Sendable { await backing.capabilities() } + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + try await backing.resolvePotentialPath(path: path) + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try ensureWritable() diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index b78bea2..816e7b9 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -466,14 +466,10 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { } var resolved = request switch request.operation { - case .stat, .remove, .readSymlink: + case .stat, .remove, .readSymlink, .exists: if let path = request.path { resolved.path = try await resolvedPath(path, followFinal: false) } - case .exists: - if let path = request.path { - resolved.path = try await resolvedPath(path, followFinal: true) - } case .move, .copy: if let path = request.sourcePath { resolved.sourcePath = try await resolvedPath(path, followFinal: false) @@ -490,7 +486,7 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { } case .glob: // Glob enumerates entries without dereferencing matched final links. - // Only its non-pattern prefix needs resolution. + // Only its non-pattern prefix needs resolution; an exact path uses lstat semantics. if let path = request.path { var prefix = WorkspacePath.root var suffix: [String] = [] @@ -501,7 +497,7 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { prefix = prefix.appending(component) } } - var target = try await resolvedPath(prefix, followFinal: true) + var target = try await resolvedPath(prefix, followFinal: !suffix.isEmpty) for component in suffix { target = target.appending(component) } diff --git a/Tests/WorkspaceTests/FileSystemFollowupTests.swift b/Tests/WorkspaceTests/FileSystemFollowupTests.swift new file mode 100644 index 0000000..9b2a79d --- /dev/null +++ b/Tests/WorkspaceTests/FileSystemFollowupTests.swift @@ -0,0 +1,273 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Filesystem follow-up contracts") +struct FileSystemFollowupTests { + @Test(arguments: [false, true]) + func `writes through dangling links resolve relative targets from physical parents`(localBackend: Bool) async throws { + let root = try TestSupport.temporaryDirectory("PhysicalDanglingWrite") + defer { TestSupport.remove(root) } + let base: any FileSystem = localBackend ? try LocalFileSystem(root: root) : InMemoryFileSystem() + try await preparePhysicalDanglingLink(on: base) + try await base.writeFile(path: "/a/link/dangle", data: Data(count: 120), append: false) + #expect(try await base.readFile(path: "/a/link/dangle") == Data(count: 120)) + #expect(try await base.readFile(path: "/x/t") == Data(count: 120)) + #expect(try await base.readFile(path: "/a/t") == Data(count: 100)) + } + + @Test(arguments: ["local", "memory", "authorized local"]) + func `quota charges dangling targets beneath physical parents`(backend: String) async throws { + let root = try TestSupport.temporaryDirectory("PhysicalDanglingQuota") + defer { TestSupport.remove(root) } + let base: any FileSystem = backend == "memory" ? InMemoryFileSystem() : try LocalFileSystem(root: root) + try await preparePhysicalDanglingLink(on: base) + let decorated: any FileSystem = backend == "authorized local" + ? AuthorizedFileSystem(base: base, authorizer: PermissionAuthorizer { _ in .allow }) : base + let limited = LimitedFileSystem(base: decorated, limits: .init(maxTotalBytes: 150)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 220, limit: 150)) { + try await limited.writeFile(path: "/a/link/dangle", data: Data(count: 120), append: false) + } + #expect(!(await base.exists(path: "/x/t"))) + #expect(try await base.readFile(path: "/a/t") == Data(count: 100)) + } + + static var absoluteTargetBackends: [String] { + var backends = ["local", "limited", "sandbox", "mounted", "limited mounted"] + #if os(macOS) + backends.append("security scoped") + #endif + return backends + } + + @Test(arguments: absoluteTargetBackends) + func `absolute host dangling targets retain backend namespaces through decorators`(backend: String) async throws { + let root = try TestSupport.temporaryDirectory("AbsoluteDanglingQuota") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createSymlink(path: "/abs", target: root.resolvingSymlinksInPath().appendingPathComponent("target").path) + let mounted = MountedFileSystem(mounts: [.init(mountPoint: "/mounted", fileSystem: local)]) + let base: any FileSystem + switch backend { + case "limited": base = LimitedFileSystem(base: local, limits: .init()) + case "sandbox": base = try SandboxFileSystem(root: .url(root)) + case "mounted": base = mounted + case "limited mounted": base = LimitedFileSystem(base: mounted, limits: .init()) + #if os(macOS) + case "security scoped": base = try SecurityScopedFileSystem(url: root) + #endif + default: base = local + } + let mountedBackend = backend.contains("mounted") + let limited = LimitedFileSystem(base: base, limits: .init(maxEntryCount: mountedBackend ? 3 : 2)) + try await limited.writeFile(path: mountedBackend ? "/mounted/abs" : "/abs", data: Data([1]), append: false) + #expect(try await local.stat(path: "/abs").kind == .symlink) + #expect(try await local.readFile(path: "/target") == Data([1])) + #expect(try await local.listDirectory(path: "/").map(\.name) == ["abs", "target"]) + } + + @Test + func `absolute dangling targets count missing parents without creating them during preflight`() async throws { + let root = try TestSupport.temporaryDirectory("AbsoluteDanglingParents") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createSymlink(path: "/abs", target: root.resolvingSymlinksInPath().appendingPathComponent("new/target").path) + let tooSmall = LimitedFileSystem(base: local, limits: .init(maxEntryCount: 2)) + await #expect(throws: FileSystemLimitError.entryCount(attempted: 3, limit: 2)) { + try await tooSmall.writeFile(path: "/abs", data: Data([1]), append: false) + } + #expect(!(await local.exists(path: "/new"))) + let enough = LimitedFileSystem(base: local, limits: .init(maxEntryCount: 3)) + try await enough.writeFile(path: "/abs", data: Data([1]), append: false) + #expect(try await local.readFile(path: "/new/target") == Data([1])) + #expect(try await local.stat(path: "/abs").kind == .symlink) + } + + @Test + func `local quota permits appending among many equal size files`() async throws { + let root = try TestSupport.temporaryDirectory("LocalQuotaPrecision") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + for index in 0..<20 { + try await local.createFile(path: .root.appending("file-\(index)"), data: Data()) + } + let limited = LimitedFileSystem(base: local, limits: .init(maxTotalBytes: 1)) + try await limited.writeFile(path: "/file-0", data: Data([1]), append: true) + #expect(try await local.readFile(path: "/file-0") == Data([1])) + #expect(try await local.stat(path: "/file-1").size == 0) + } + + @Test + func `local metadata identifies hard links without treating directory links as aliases`() async throws { + let root = try TestSupport.temporaryDirectory("LocalMetadata") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/dir/child", recursive: true) + try await local.createFile(path: "/file", data: Data("ab".utf8)) + try await local.createHardLink(path: "/alias", target: "/file") + try await local.createSymlink(path: "/link", target: "file") + let original = try await local.stat(path: "/file") + let identity = try #require(original.identity) + #expect(original.linkCount == 2) + #expect(try await local.stat(path: "/alias").identity == identity) + #expect(try await local.stat(path: "/link").identity != identity) + #expect(try await local.stat(path: "/link").linkCount == 1) + #expect(try await local.stat(path: "/dir").linkCount == nil) + for entry in try await local.listDirectory(path: "/") { + let info = try await local.stat(path: entry.info.path) + #expect(entry.info.identity == info.identity) + #expect(entry.info.linkCount == info.linkCount) + } + let limited = LimitedFileSystem(base: local, limits: .init(maxTotalBytes: 6)) + try await limited.writeFile(path: "/alias", data: Data("c".utf8), append: true) + #expect(try await local.readFile(path: "/file") == Data("abc".utf8)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 8, limit: 6)) { + try await limited.writeFile(path: "/file", data: Data("d".utf8), append: true) + } + #expect(try await local.readFile(path: "/alias") == Data("abc".utf8)) + } + + @Test + func `separate local instances mounted twice account for the same inode`() async throws { + let root = try TestSupport.temporaryDirectory("LocalMountedAliases") + defer { TestSupport.remove(root) } + let first = try LocalFileSystem(root: root) + let second = try LocalFileSystem(root: root) + for index in 0..<10 { + try await first.createFile(path: .root.appending("file-\(index)"), data: Data("ab".utf8)) + } + let mounted = MountedFileSystem(mounts: [ + .init(mountPoint: "/first", fileSystem: first), + .init(mountPoint: "/second", fileSystem: second), + ]) + let identity = try #require(try await mounted.stat(path: "/first/file-0").identity) + #expect(try await mounted.stat(path: "/second/file-0").identity == identity) + #expect(try await mounted.stat(path: "/first/file-0").linkCount == 1) + let limited = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 42)) + try await limited.writeFile(path: "/first/file-0", data: Data("c".utf8), append: true) + #expect(try await mounted.readFile(path: "/second/file-0") == Data("abc".utf8)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 44, limit: 42)) { + try await limited.writeFile(path: "/second/file-0", data: Data("d".utf8), append: true) + } + #expect(try await first.readFile(path: "/file-0") == Data("abc".utf8)) + } + + @Test + func `local replacement writes follow final symlinks and charge dangling targets`() async throws { + let root = try TestSupport.temporaryDirectory("LocalLinkReplacement") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createFile(path: "/file", data: Data("original".utf8)) + try await local.createSymlink(path: "/link", target: "file") + #expect(try await local.readFile(path: "/link") == Data("original".utf8)) + try await local.writeFile(path: "/link", data: Data("replacement".utf8), append: false) + #expect(try await local.stat(path: "/link").kind == .symlink) + #expect(try await local.readSymlink(path: "/link") == "file") + #expect(try await local.readFile(path: "/file") == Data("replacement".utf8)) + + try await local.remove(path: "/link", recursive: false) + try await local.remove(path: "/file", recursive: false) + try await local.createSymlink(path: "/dangling", target: "missing") + let oneEntry = LimitedFileSystem(base: local, limits: .init(maxEntryCount: 1)) + await #expect(throws: FileSystemLimitError.entryCount(attempted: 2, limit: 1)) { + try await oneEntry.writeFile(path: "/dangling", data: Data("new".utf8), append: false) + } + #expect(!(await local.exists(path: "/missing"))) + let twoEntries = LimitedFileSystem(base: local, limits: .init(maxTotalBytes: 3, maxEntryCount: 2)) + try await twoEntries.writeFile(path: "/dangling", data: Data("new".utf8), append: false) + #expect(try await local.stat(path: "/dangling").kind == .symlink) + #expect(try await local.readSymlink(path: "/dangling") == "missing") + #expect(try await local.readFile(path: "/missing") == Data("new".utf8)) + } + + @Test(arguments: [false, true]) + func `overlay dangling links survive snapshots and persisted checkpoint restoration`(upperLayer: Bool) async throws { + let root = try TestSupport.temporaryDirectory("OverlayDangling") + let history = try TestSupport.temporaryDirectory("OverlayDanglingHistory") + defer { TestSupport.remove(root); TestSupport.remove(history) } + let local = try LocalFileSystem(root: root) + let overlay = OverlayFileSystem(over: local) + let layer: any FileSystem = upperLayer ? overlay : local + try await layer.createSymlink(path: "/dangling", target: "missing") + #expect(await overlay.exists(path: "/dangling")) + #expect(try await overlay.glob(pattern: "/dangling", currentDirectory: "/") == ["/dangling"]) + let id = UUID() + let workspace = Workspace(workspaceID: id, fileSystem: overlay, persistence: .directory(history)) + let checkpoint = try await workspace.createCheckpoint() + let snapshot = try await Snapshot.capture(from: overlay) + try await overlay.remove(path: "/dangling", recursive: false) + #expect(!(await overlay.exists(path: "/dangling"))) + try await Snapshot.restore(snapshot, to: overlay) + #expect(try await overlay.readSymlink(path: "/dangling") == "missing") + + try await overlay.remove(path: "/dangling", recursive: false) + let reopened = Workspace(workspaceID: id, fileSystem: overlay, persistence: .directory(history)) + #expect(await reopened.exists("/dangling", at: .checkpoint(checkpoint.id))) + _ = try await reopened.restore(to: checkpoint) + #expect(try await overlay.readSymlink(path: "/dangling") == "missing") + #expect(!(await overlay.exists(path: "/missing"))) + } + + @Test + func `all backends use lstat existence through symlinked parents`() async throws { + let root = try TestSupport.temporaryDirectory("BackendLinkExistence") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let memory = InMemoryFileSystem() + for base: any FileSystem in [local, memory] { + try await base.createDirectory(path: "/dir", recursive: false) + try await base.createSymlink(path: "/dir/dangling", target: "../missing") + try await base.createSymlink(path: "/parent", target: "dir") + try await base.createSymlink(path: "/loop", target: "loop") + } + var backends: [any FileSystem] = [ + local, memory, OverlayFileSystem(over: local), + try SandboxFileSystem(root: .url(root)), + LimitedFileSystem(base: local, limits: .init()), + MountedFileSystem(base: local), + AuthorizedFileSystem(base: local, authorizer: PermissionAuthorizer { _ in .allow }), + ] + #if os(macOS) + backends.append(try SecurityScopedFileSystem(url: root)) + #endif + for backend in backends { + #expect(await backend.exists(path: "/parent/dangling")) + #expect(try await backend.glob(pattern: "/parent/dangling", currentDirectory: "/") == ["/parent/dangling"]) + #expect(await backend.exists(path: "/loop")) + #expect(!(await backend.exists(path: "/parent/dangling/child"))) + #expect(!(await backend.exists(path: "/missing"))) + } + } + + @Test + func `disk adapters preserve identity link count and features`() async throws { + let root = try TestSupport.temporaryDirectory("AdapterMetadata") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createFile(path: "/file", data: Data([1])) + try await local.createHardLink(path: "/alias", target: "/file") + let identity = try #require(try await local.stat(path: "/file").identity) + var adapters: [any FileSystem] = [try SandboxFileSystem(root: .url(root))] + #if os(macOS) + adapters.append(try SecurityScopedFileSystem(url: root, mode: .readOnly)) + adapters.append(try SecurityScopedFileSystem(url: root, mode: .readWrite)) + #endif + for adapter in adapters { + #expect(await adapter.capabilities() == local.capabilities()) + #expect(try await adapter.stat(path: "/file").identity == identity) + #expect(try await adapter.stat(path: "/file").linkCount == 2) + for entry in try await adapter.listDirectory(path: "/") { + #expect(entry.info.identity == identity) + #expect(entry.info.linkCount == 2) + } + } + } +} + +private func preparePhysicalDanglingLink(on base: any FileSystem) async throws { + try await base.createDirectory(path: "/a", recursive: false) + try await base.createDirectory(path: "/x/y", recursive: true) + try await base.createSymlink(path: "/a/link", target: "../x/y") + try await base.createSymlink(path: "/x/y/dangle", target: "../t") + try await base.createFile(path: "/a/t", data: Data(count: 100)) +} diff --git a/Tests/WorkspaceTests/FileSystemRegressionTests.swift b/Tests/WorkspaceTests/FileSystemRegressionTests.swift index c87aabd..fbac3e0 100644 --- a/Tests/WorkspaceTests/FileSystemRegressionTests.swift +++ b/Tests/WorkspaceTests/FileSystemRegressionTests.swift @@ -185,7 +185,7 @@ struct FileSystemRegressionTests { } @Test - func `unknown identities conservatively protect preexisting disk aliases`() async throws { + func `preexisting disk hard links charge every path and replacement breaks sharing`() async throws { let root = try TestSupport.temporaryDirectory("LimitedDiskAliases") defer { TestSupport.remove(root) } let disk = try LocalFileSystem(root: root) @@ -198,6 +198,8 @@ struct FileSystemRegressionTests { #expect(try await disk.readFile(path: "/file") == Data("ab".utf8)) try await limited.writeFile(path: "/alias", data: Data("abc".utf8), append: false) #expect(try await disk.stat(path: "/file").size == 2) + #expect(try await disk.readFile(path: "/alias") == Data("abc".utf8)) + #expect(try await disk.stat(path: "/alias").identity != disk.stat(path: "/file").identity) } @Test @@ -298,13 +300,14 @@ struct FileSystemRegressionTests { defer { TestSupport.remove(root) } let disk = try LocalFileSystem(root: root) try await disk.writeFile(path: "/file", data: Data("ab".utf8), append: false) - try await disk.createSymlink(path: "/link", target: "/file") + try await disk.createSymlink(path: "/link", target: "file") let diskLimited = LimitedFileSystem(base: disk, limits: .init(maxTotalBytes: 3)) - await #expect(throws: FileSystemLimitError.totalBytes(attempted: 5, limit: 3)) { - try await diskLimited.writeFile(path: "/link", data: Data("xyz".utf8), append: false) + try await diskLimited.writeFile(path: "/link", data: Data("xyz".utf8), append: false) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 4, limit: 3)) { + try await diskLimited.writeFile(path: "/link", data: Data("wxyz".utf8), append: false) } #expect(try await disk.stat(path: "/link").kind == .symlink) - #expect(try await disk.readFile(path: "/file") == Data("ab".utf8)) + #expect(try await disk.readFile(path: "/file") == Data("xyz".utf8)) } @Test From f48d987a523129cdc78981a90dbbc0386525f140 Mon Sep 17 00:00:00 2001 From: Zac White Date: Thu, 8 Oct 2026 21:10:11 -0700 Subject: [PATCH 10/34] Bound diff cost, speed up checkpoint paging, and budget remaining tool output TextDiff trims common prefixes and suffixes and runs a bounded Myers search (up to 1,000 edits or 1,000,000 steps, with a separate budget per intra-line pair), falling back to one replacement hunk with exact statistics only for genuinely large edit distances. Checkpointing a 100k-line new file no longer takes tens of seconds. The in-memory store caches revision indexes per immutable snapshot, and checkpoint pages resolve once and use ranged reads, so paging through a checkpoint file no longer re-hashes the whole snapshot on every page. Checkpoint reads resolve intermediate symlinks within the workspace, and directory-backed capture keeps dangling and cyclic links. workspace_glob, workspace_diff and workspace_apply results are bounded by encoded-size budgets with paging and explicit truncation fields, accounted incrementally. workspace_apply never reports failure after its edits have committed; it falls back to a minimal summary instead. workspace_glob now returns an object ({matches, totalMatches, ...}) rather than a bare array. --- README.md | 45 ++- Sources/Workspace/CheckpointStore.swift | 18 +- Sources/Workspace/RevisionIndex.swift | 30 ++ Sources/Workspace/TextDiff.swift | 215 ++++++++++---- Sources/Workspace/WorkspaceChanges.swift | 2 + Sources/Workspace/WorkspaceRevision.swift | 62 ++--- Sources/Workspace/WorkspaceToolAdapter.swift | 111 ++++++-- Sources/Workspace/WorkspaceToolBudget.swift | 157 +++++++++++ .../FollowupPerformanceTests.swift | 160 +++++++++++ Tests/WorkspaceTests/TextDiffTests.swift | 88 ++++++ .../WorkspaceToolAdapterTests.swift | 262 +++++++++++++++++- 11 files changed, 1009 insertions(+), 141 deletions(-) create mode 100644 Sources/Workspace/WorkspaceToolBudget.swift create mode 100644 Tests/WorkspaceTests/FollowupPerformanceTests.swift diff --git a/README.md b/README.md index 216797b..e978663 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,14 @@ let changes = try await workspace.diff( ) ``` -Checkpoint reads use manifest metadata and load only selected blobs. `TextDiff` includes total line counts, addition/deletion statistics, hunks, and UTF-16 intra-line ranges. +Checkpoint reads use cached in-memory indexes or persisted manifest metadata, resolve directory +symlinks inside the workspace namespace (at most 64 hops), and read selected byte ranges without +materializing whole blobs. `TextDiff` includes total line counts, addition/deletion statistics, +hunks, and UTF-16 intra-line ranges. Empty and identical inputs and unchanged prefixes/suffixes +avoid the general diff search. A Myers search preserves sparse edits; if its work or edit-distance +limit is reached, the unmatched interior uses a coarse replacement hunk. Its statistics describe +the emitted additions/deletions and may exceed a minimal edit script. The line search and each +intra-line pair have independent limits of 1,000 edits or 1,000,000 visited search steps. Use `Workspace.Archive` only when a fully materialized, portable subtree is actually required: @@ -307,6 +314,42 @@ All mutating adapter calls use the Workspace edit/checkpoint APIs, so they retai event, transaction, authorization, and limit behavior as direct callers. The diff result carries the structured `ChangeSet` and can optionally include unified-patch text for prompt or CLI interop. +Tool arguments use snake_case and results use camelCase. Output budgets for glob, diff, and apply +count **encoded JSON bytes**, including escaping and metadata: `max_bytes` defaults to 65,536, +has a minimum of 1,024, and clamps to 262,144. Diff and apply also accept `max_file_bytes` for each +file's encoded text diff plus optional patch; it defaults to 8,192, clamps to 65,536, and accepts 0 +to omit all text detail. Returned patch sections are complete; text that cannot fit is omitted +with `textDiffOmissions.reason = "outputBudgetExceeded"`. + +| Tool | Additional arguments | Result | +| --- | --- | --- | +| `workspace_glob` | `match_offset` (default 0), `max_matches` (default 1,000, cap 10,000), `max_bytes` | `{matches, totalMatches, matchOffset, nextMatchOffset?, omittedMatches, truncated}` | +| `workspace_diff` | `change_offset` (default 0), `paths` (optional exact paths), `max_bytes`, `max_file_bytes` | `{changes, statistics, totalChanges, changeOffset, nextChangeOffset?, omittedFiles, omittedTextDiffs, truncated, unifiedPatch?}` | +| `workspace_apply` | `max_bytes`, `max_file_bytes` | `{changes, failures, statistics, totalChanges, changeOffset: 0, omittedFiles, omittedTextDiffs, truncated}` | + +Continue glob/diff at the returned offset, keeping the same pattern/paths and revisions. Pin both +diff revisions to checkpoints for stable paging while the workspace changes. Narrow glob patterns +or diff `paths`, or raise `max_file_bytes`, to retrieve detail; use `workspace_read` for full text. +`max_text_bytes` skips text diffing for files larger than its source-byte limit (default and cap +1,000,000 bytes per input file); `max_file_bytes` bounds encoded diff bytes per file in the result. +The nested `changes` contains only returned detail; top-level `statistics` describes all selected +changes using available computed text diffs. `omittedFiles` counts change entries at/after the +requested offset that are absent from this response, including entries on subsequent pages. +`omittedTextDiffs` counts returned entries whose computed diff was removed by the output budget. +Glob skips a single path too large for a page and increments `omittedMatches`; diff similarly +counts oversized path metadata in `omittedFiles`. Offsets still advance past those entries. + +Apply always completes the full atomic edit operation before bounding its response. Do not repeat +an apply call to retrieve omitted detail. Use `workspace_read` to inspect omitted files, or create +a checkpoint before applying and diff from it. If response encoding fails after commit, apply +returns a minimal success envelope with empty `changes`, full statistics, and `truncated: true`. + +Beta compatibility: glob tool results now use an object instead of a bare array. Apply and diff +retain their existing keys and add budget/paging metadata; large results can contain less detail. +Direct Workspace method signatures are unchanged. Public adapter budget constants and +`TextDiffOmission.Reason.outputBudgetExceeded` are additive; exhaustive switches on the reason +must handle the new case. + ## Testing ```bash diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 351f254..1134202 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -60,6 +60,8 @@ extension CheckpointStore { actor InMemoryCheckpointStore: CheckpointStore { private var checkpointsByWorkspace: [UUID: [Checkpoint]] = [:] private var snapshotsByWorkspace: [UUID: [UUID: Snapshot]] = [:] + private(set) var revisionIndexBuildCount = 0 + private var revisionIndexesByWorkspace: [UUID: [UUID: RevisionIndex]] = [:] private var mutationsByWorkspace: [UUID: [Mutation]] = [:] private var mutationHighWaterMarks: [UUID: Int] = [:] private var mutationRetractionEpochs: [UUID: Int] = [:] @@ -102,6 +104,7 @@ actor InMemoryCheckpointStore: CheckpointStore { parentSnapshot: parentSnapshot ) snapshotsByWorkspace[draft.workspaceID, default: [:]][snapshot.id] = snapshot + revisionIndexesByWorkspace[draft.workspaceID]?[snapshot.id] = nil list.append(checkpoint) checkpointsByWorkspace[draft.workspaceID] = list.sorted(by: Checkpoint.orderedBefore) return checkpoint @@ -125,7 +128,12 @@ actor InMemoryCheckpointStore: CheckpointStore { } func loadRevisionIndex(id: UUID, workspaceId: UUID) async throws -> RevisionIndex? { - snapshotsByWorkspace[workspaceId]?[id].map(RevisionIndex.init) + if let cached = revisionIndexesByWorkspace[workspaceId]?[id] { return cached } + guard let snapshot = snapshotsByWorkspace[workspaceId]?[id] else { return nil } + let index = RevisionIndex(snapshot: snapshot) + revisionIndexBuildCount += 1 + revisionIndexesByWorkspace[workspaceId, default: [:]][id] = index + return index } func readSnapshotFile( @@ -220,6 +228,9 @@ actor InMemoryCheckpointStore: CheckpointStore { if !dryRun { checkpointsByWorkspace[workspaceId] = retained snapshotsByWorkspace[workspaceId] = retainedSnapshots + revisionIndexesByWorkspace[workspaceId] = revisionIndexesByWorkspace[workspaceId]?.filter { + retainedSnapshotIDs.contains($0.key) + } } return StoreCompactionResult(report: report, checkpoints: retained) } @@ -506,10 +517,11 @@ actor FileCheckpointStore: CheckpointStore { previousCaptureStartedAt: Date?, workspaceId: UUID ) async throws -> SnapshotManifest.Node { - guard await filesystem.exists(path: path) else { + // exists follows the final symlink on some backends, losing dangling + // links and cycles. Capture the entry itself, just as Snapshot.capture does. + guard let info = try await Snapshot.infoIfPresent(at: path, on: filesystem) else { return SnapshotManifest.Node(kind: .missing, path: path) } - let info = try await filesystem.stat(path: path) switch info.kind { case .file: // Conservatively cover second-resolution and two-second-resolution timestamps. diff --git a/Sources/Workspace/RevisionIndex.swift b/Sources/Workspace/RevisionIndex.swift index 1e02c73..4c1814f 100644 --- a/Sources/Workspace/RevisionIndex.swift +++ b/Sources/Workspace/RevisionIndex.swift @@ -77,6 +77,36 @@ struct RevisionIndex: Sendable, Equatable { Self.entry(at: path, in: entry) } + /// Resolve each component in the revision namespace, including directory symlinks. + func readableFile(at path: WorkspacePath) throws -> (path: WorkspacePath, size: UInt64) { + var pending = path.components + var current = WorkspacePath.root + var hops = 0 + while !pending.isEmpty { + let component = pending.removeFirst() + current = current.appending(component) + guard let entry = entry(at: current) else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) + } + switch entry { + case let .symbolicLink(_, target, _): + guard hops < 64 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } + hops += 1 + let targetPath = try WorkspacePath(target, relativeTo: current.parent) + pending = targetPath.components + pending + current = .root + case let .file(_, size, _, _): + guard pending.isEmpty else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) } + return (current, size) + case .directory: + continue + case .missing: + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) + } + } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) + } + private static func entry(at path: WorkspacePath, in entry: Entry) -> Entry? { if entry.path == path { return entry } guard case let .directory(_, _, children) = entry else { return nil } diff --git a/Sources/Workspace/TextDiff.swift b/Sources/Workspace/TextDiff.swift index 3f46234..17f61f1 100644 --- a/Sources/Workspace/TextDiff.swift +++ b/Sources/Workspace/TextDiff.swift @@ -91,8 +91,8 @@ public struct TextDiff: Sendable, Equatable, Codable { Statistics( originalLineCount: originalLineCount, updatedLineCount: updatedLineCount, - additions: hunks.flatMap(\.lines).count { $0.kind == .added }, - deletions: hunks.flatMap(\.lines).count { $0.kind == .removed } + additions: hunks.reduce(0) { $0 + $1.lines.count { $0.kind == .added } }, + deletions: hunks.reduce(0) { $0 + $1.lines.count { $0.kind == .removed } } ) } @@ -103,19 +103,41 @@ public struct TextDiff: Sendable, Equatable, Codable { } /// Builds a line diff with three context lines and UTF-16 intra-line ranges. + /// A bounded Myers search retains precise sparse edits even in large files. + /// When its work or edit-distance limit is reached, the unmatched interior + /// becomes a coarse replacement. Statistics then describe a nonminimal script. public static func lineBased(from originalContent: String, to updatedContent: String) -> TextDiff { + var work = Work() + return lineBased(from: originalContent, to: updatedContent, work: &work) + } + + /// Internal diagnostics keep work limits observable without changing the diff's + /// public representation, equality, or encoding. + struct SearchWork { + var steps = 0 + var editDistance = 0 + var usedTrivialCase = false + var usedCoarseFallback = false + } + + struct Work { + var lines = SearchWork() + var intralineSearches = 0 + var maximumIntralineSteps = 0 + var intralineFallbacks = 0 + } + + static let maximumSearchSteps = 1_000_000 + static let maximumEditDistance = 1_000 + + static func lineBased(from originalContent: String, to updatedContent: String, work: inout Work) -> TextDiff { let originalLines = diffLineTokens(in: originalContent) + if originalContent == updatedContent { + work.lines.usedTrivialCase = true + return TextDiff(originalLineCount: originalLines.count, updatedLineCount: originalLines.count, hunks: []) + } let updatedLines = diffLineTokens(in: updatedContent) - let changes = Array(updatedLines.difference(from: originalLines)) - - let removals = Dictionary(grouping: changes.compactMap { change -> (Int, DiffLineToken)? in - guard case let .remove(offset, element, _) = change else { return nil } - return (offset, element) - }, by: \.0) - let insertions = Dictionary(grouping: changes.compactMap { change -> (Int, DiffLineToken)? in - guard case let .insert(offset, element, _) = change else { return nil } - return (offset, element) - }, by: \.0) + let (removals, insertions) = boundedDifferenceOffsets(from: originalLines, to: updatedLines, work: &work.lines) var lines: [Line] = [] var originalIndex = 0 @@ -124,34 +146,20 @@ public struct TextDiff: Sendable, Equatable, Codable { var updatedLineNumber = 1 while originalIndex < originalLines.count || updatedIndex < updatedLines.count { - if let removed = removals[originalIndex] { - for (_, token) in removed { - lines.append( - Line( - kind: .removed, - text: token.text, - hasTrailingNewline: token.hasTrailingNewline, - oldLineNumber: originalLineNumber - ) - ) - originalIndex += 1 - originalLineNumber += 1 - } + if removals.contains(originalIndex) { + let token = originalLines[originalIndex] + lines.append(Line(kind: .removed, text: token.text, hasTrailingNewline: token.hasTrailingNewline, + oldLineNumber: originalLineNumber)) + originalIndex += 1 + originalLineNumber += 1 continue } - if let inserted = insertions[updatedIndex] { - for (_, token) in inserted { - lines.append( - Line( - kind: .added, - text: token.text, - hasTrailingNewline: token.hasTrailingNewline, - newLineNumber: updatedLineNumber - ) - ) - updatedIndex += 1 - updatedLineNumber += 1 - } + if insertions.contains(updatedIndex) { + let token = updatedLines[updatedIndex] + lines.append(Line(kind: .added, text: token.text, hasTrailingNewline: token.hasTrailingNewline, + newLineNumber: updatedLineNumber)) + updatedIndex += 1 + updatedLineNumber += 1 continue } guard originalIndex < originalLines.count, updatedIndex < updatedLines.count else { break } @@ -171,7 +179,7 @@ public struct TextDiff: Sendable, Equatable, Codable { updatedLineNumber += 1 } - applyIntralineRanges(to: &lines) + applyIntralineRanges(to: &lines, work: &work) let changedIndices = lines.indices.filter { lines[$0].kind != .context } guard !changedIndices.isEmpty else { return TextDiff(originalLineCount: originalLines.count, updatedLineCount: updatedLines.count, hunks: []) @@ -230,7 +238,7 @@ private func diffLineTokens(in text: String) -> [DiffLineToken] { return tokens } -private func applyIntralineRanges(to lines: inout [TextDiff.Line]) { +private func applyIntralineRanges(to lines: inout [TextDiff.Line], work: inout TextDiff.Work) { var index = 0 while index < lines.count { guard lines[index].kind != .context else { @@ -244,7 +252,11 @@ private func applyIntralineRanges(to lines: inout [TextDiff.Line]) { for pairIndex in 0.. (old: [TextDiff.IntralineRange], new: [TextDiff.IntralineRange]) { let oldCharacters = Array(original) let newCharacters = Array(updated) - let difference = newCharacters.difference(from: oldCharacters) - let removed = difference.compactMap { change -> Int? in - guard case let .remove(offset, _, _) = change else { return nil } - return offset - } - let inserted = difference.compactMap { change -> Int? in - guard case let .insert(offset, _, _) = change else { return nil } - return offset - } + let (removed, inserted) = boundedDifferenceOffsets(from: oldCharacters, to: newCharacters, work: &work) + return ( - collapsedUTF16Ranges(characterOffsets: removed, in: original), - collapsedUTF16Ranges(characterOffsets: inserted, in: updated) + collapsedUTF16Ranges(characterOffsets: Array(removed), in: original), + collapsedUTF16Ranges(characterOffsets: Array(inserted), in: updated) ) } @@ -284,12 +290,103 @@ private func collapsedUTF16Ranges(characterOffsets: [Int], in text: String) -> [ characterRanges.append(offset..<(offset + 1)) } } + var utf16Offsets = [0] + for character in characters { utf16Offsets.append(utf16Offsets.last! + character.utf16.count) } return characterRanges.map { range in - let prefix = String(characters[..( + from old: [Element], to new: [Element], work: inout TextDiff.SearchWork +) -> (removed: Set, inserted: Set) { + var prefix = 0 + while prefix < min(old.count, new.count), old[prefix] == new[prefix] { prefix += 1 } + var oldEnd = old.count + var newEnd = new.count + while oldEnd > prefix, newEnd > prefix, old[oldEnd - 1] == new[newEnd - 1] { + oldEnd -= 1 + newEnd -= 1 + } + let oldCount = oldEnd - prefix + let newCount = newEnd - prefix + func replacement() -> (Set, Set) { (Set(prefix.. 0, newCount > 0 else { + work.usedTrivialCase = true + return replacement() + } + // Length imbalance is a lower bound on D, so no search can fit this limit. + guard abs(oldCount - newCount) <= TextDiff.maximumEditDistance else { + work.usedCoarseFallback = true + return replacement() + } + + var trace: [[Int]] = [] + for depth in 0...min(oldCount + newCount, TextDiff.maximumEditDistance) { + let previous = trace.last ?? [] + var frontier = Array(repeating: 0, count: depth * 2 + 1) + for diagonal in stride(from: -depth, through: depth, by: 2) { + guard work.steps < TextDiff.maximumSearchSteps else { + work.usedCoarseFallback = true + return replacement() + } + work.steps += 1 + var x: Int + if depth == 0 { + x = 0 + } else if diagonal == -depth || (diagonal != depth && + previous[diagonal - 1 + depth - 1] < previous[diagonal + 1 + depth - 1]) { + x = previous[diagonal + 1 + depth - 1] + } else { + x = previous[diagonal - 1 + depth - 1] + 1 + } + var y = x - diagonal + while x < oldCount, y < newCount { + guard work.steps < TextDiff.maximumSearchSteps else { + work.usedCoarseFallback = true + return replacement() + } + work.steps += 1 + guard old[prefix + x] == new[prefix + y] else { break } + x += 1 + y += 1 + } + frontier[diagonal + depth] = x + if x >= oldCount, y >= newCount { + work.editDistance = depth + var removed: Set = [] + var inserted: Set = [] + // Recover the chosen shortest script through earlier frontiers. + for d in stride(from: depth, through: 1, by: -1) { + let previous = trace[d - 1] + let k = x - y + let previousK: Int + if k == -d || (k != d && previous[k - 1 + d - 1] < previous[k + 1 + d - 1]) { + previousK = k + 1 + } else { + previousK = k - 1 + } + let previousX = previous[previousK + d - 1] + let previousY = previousX - previousK + while x > previousX, y > previousY { x -= 1; y -= 1 } + if x == previousX { + y -= 1 + inserted.insert(prefix + y) + } else { + x -= 1 + removed.insert(prefix + x) + } + } + return (removed, inserted) + } + } + trace.append(frontier) } + work.usedCoarseFallback = true + return replacement() } diff --git a/Sources/Workspace/WorkspaceChanges.swift b/Sources/Workspace/WorkspaceChanges.swift index b8eb923..edee407 100644 --- a/Sources/Workspace/WorkspaceChanges.swift +++ b/Sources/Workspace/WorkspaceChanges.swift @@ -36,6 +36,8 @@ public struct TextDiffOmission: Sendable, Codable, Equatable { public enum Reason: String, Sendable, Codable { case binary case sizeLimitExceeded + /// The tool adapter omitted text to fit its encoded output budget. + case outputBudgetExceeded } public var path: WorkspacePath diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index c2d4b6b..1e0b1f1 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -123,31 +123,17 @@ extension Workspace { offset: UInt64 = 0, length: Int? = nil ) async throws -> Data { - let resolved = try await resolvedRevision(revision) - var current = path - for _ in 0..<64 { - guard let entry = resolved.index.entry(at: current) else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) - } - switch entry { - case .file: - return try await resolved.readRange(current, offset, length) - case let .symbolicLink(_, target, _): - current = WorkspacePath(normalizing: target, relativeTo: current.parent) - case .directory: - throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) - case .missing: - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) - } - } - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) + let file = try await resolvedReadableFile(at: path, revision: revision) + return try await file.readRange(offset, length) + } + + struct ResolvedReadableFile: Sendable { + var size: UInt64 + var readRange: @Sendable (UInt64, Int?) async throws -> Data } - /// Returns a read path and its size without loading file contents. Current paths - /// use filesystem resolution; stat and readData must use that same resolved path. - /// Checkpoint symlinks retain revisionData's normalization and POSIX errors. - func revisionFileSize(at path: WorkspacePath, revision: Revision) async throws -> (resolvedPath: WorkspacePath, size: UInt64) { - let index: RevisionIndex + /// Resolve once for metadata and all reads needed to decode one UTF-8 page. + func resolvedReadableFile(at path: WorkspacePath, revision: Revision) async throws -> ResolvedReadableFile { switch revision { case .current: let resolvedPath = try await filesystem.resolveRealPath(path: path) @@ -155,29 +141,17 @@ extension Workspace { guard metadata.kind == .file else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) } - return (resolvedPath, metadata.size) + let filesystem = filesystem + return ResolvedReadableFile(size: metadata.size, readRange: { offset, length in + try await filesystem.readFile(path: resolvedPath, offset: offset, length: length) + }) case .checkpoint: - index = try await resolvedRevision(revision).index - } - var current = path - for _ in 0..<64 { - guard let entry = index.entry(at: current) else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) - } - if case .missing = entry { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) - } - let metadata = Self.info(for: entry) - if case let .symbolicLink(_, target, _) = entry { - current = WorkspacePath(normalizing: target, relativeTo: current.parent) - continue - } - guard metadata.kind == .file else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) - } - return (current, metadata.size) + let resolved = try await resolvedRevision(revision) + let file = try resolved.index.readableFile(at: path) + return ResolvedReadableFile(size: file.size, readRange: { offset, length in + try await resolved.readRange(file.path, offset, length) + }) } - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } public func info(_ path: WorkspacePath, at revision: Revision = .current) async throws -> FileInfo { diff --git a/Sources/Workspace/WorkspaceToolAdapter.swift b/Sources/Workspace/WorkspaceToolAdapter.swift index 50c4123..c32091a 100644 --- a/Sources/Workspace/WorkspaceToolAdapter.swift +++ b/Sources/Workspace/WorkspaceToolAdapter.swift @@ -72,6 +72,20 @@ public actor WorkspaceToolAdapter { public nonisolated static let defaultSearchMaxBytes = 65_536 public nonisolated static let minimumSearchMaxBytes = SearchRequest.Limits.minimumOutputBytes public nonisolated static let maximumSearchMaxBytes = 262_144 + /// Glob, diff, and apply budgets include JSON escaping and result metadata. + public nonisolated static let defaultResultMaxBytes = 65_536 + public nonisolated static let minimumResultMaxBytes = 1_024 + public nonisolated static let maximumResultMaxBytes = 262_144 + public nonisolated static let defaultFileDetailMaxBytes = 8_192 + public nonisolated static let maximumFileDetailMaxBytes = 65_536 + public nonisolated static let defaultGlobMaxMatches = 1_000 + public nonisolated static let maximumGlobMaxMatches = 10_000 + private nonisolated static var resultBudgetSchema: JSONValue { + .integerSchema("Maximum encoded result bytes; default \(defaultResultMaxBytes), minimum \(minimumResultMaxBytes). Larger values clamp to \(maximumResultMaxBytes).", minimum: minimumResultMaxBytes, maximum: maximumResultMaxBytes) + } + private nonisolated static var fileBudgetSchema: JSONValue { + .integerSchema("Maximum encoded diff bytes per file in this result, including optional patch; default \(defaultFileDetailMaxBytes). Larger values clamp to \(maximumFileDetailMaxBytes).", minimum: 0, maximum: maximumFileDetailMaxBytes) + } private nonisolated static let defaultDiffMaxBytes = DiffOptions.default.maxTextBytes ?? Int.max public nonisolated static let definitions: [WorkspaceToolDefinition] = [ @@ -90,11 +104,14 @@ public actor WorkspaceToolAdapter { ), .init( name: "workspace_glob", - description: "List workspace paths matching a glob in deterministic order.", + description: "List matching workspace paths in deterministic order within max_matches and max_bytes. If nextMatchOffset is present, call again with match_offset set to it. Narrow pattern or current_directory to reduce matches.", inputSchema: .schema( properties: [ "pattern": .stringSchema("Glob pattern."), "current_directory": .stringSchema("Directory used to resolve the pattern."), + "match_offset": .integerSchema("Matches to skip; default 0. Continue at nextMatchOffset.", minimum: 0), + "max_matches": .integerSchema("Maximum matches per page; default \(defaultGlobMaxMatches). Larger values clamp to \(maximumGlobMaxMatches).", minimum: 1, maximum: maximumGlobMaxMatches), + "max_bytes": resultBudgetSchema, "checkpoint_id": .stringSchema("Optional checkpoint UUID."), ], required: ["pattern"] @@ -120,9 +137,11 @@ public actor WorkspaceToolAdapter { ), .init( name: "workspace_apply", - description: "Atomically apply recorded text, directory, removal, copy, or move edits.", + description: "Atomically apply all edits and return bounded ChangeSet detail. Inspect truncated, omittedFiles, and omittedTextDiffs; statistics cover the full effect. Use workspace_read to inspect omitted files, or create a checkpoint before applying and diff from it. Do not repeat edits to retrieve detail.", inputSchema: .schema( properties: [ + "max_bytes": resultBudgetSchema, + "max_file_bytes": fileBudgetSchema, "edits": .arraySchema( items: .schema( properties: [ @@ -141,13 +160,17 @@ public actor WorkspaceToolAdapter { ), .init( name: "workspace_diff", - description: "Compare checkpoints or current state and return a ChangeSet plus optional unified patch.", + description: "Compare revisions and return bounded ChangeSet detail plus optional unified patch. If nextChangeOffset is present, call again with change_offset set to it using the same revisions and paths. Narrow paths or increase max_file_bytes to inspect omitted text; use workspace_read for full file contents. Inspect truncated and omittedFiles. Statistics cover all selected changes with available text diffs.", inputSchema: .schema( properties: [ + "max_bytes": resultBudgetSchema, + "max_file_bytes": fileBudgetSchema, + "change_offset": .integerSchema("Changes to skip; default 0. Continue at nextChangeOffset.", minimum: 0), + "paths": .arraySchema(items: .stringSchema("Exact workspace path to include; omit to compare all paths.")), "from_checkpoint_id": .stringSchema("Source checkpoint UUID; omit for current state."), "to_checkpoint_id": .stringSchema("Destination checkpoint UUID; omit for current state."), "unified_patch": .booleanSchema("Include git-style textual patch output."), - "max_text_bytes": .integerSchema("Maximum bytes per file for text diffing; default and cap \(defaultDiffMaxBytes).", minimum: 0, maximum: defaultDiffMaxBytes), + "max_text_bytes": .integerSchema("Skip text diffing for files larger than this many source bytes; default and cap \(defaultDiffMaxBytes).", minimum: 0, maximum: defaultDiffMaxBytes), ] ) ), @@ -171,16 +194,15 @@ public actor WorkspaceToolAdapter { minimum: Self.minimumReadMaxBytes, maximum: Self.maximumReadMaxBytes, name: "max_bytes") let offset = request.offset ?? 0 let revision = try request.revision() - let (readPath, totalBytes) = try await workspace.revisionFileSize(at: request.path, revision: revision) + let file = try await workspace.resolvedReadableFile(at: request.path, revision: revision) + let totalBytes = file.size let length = offset < totalBytes ? Int(min(UInt64(maxBytes), totalBytes - offset)) : 0 - let read = try await workspace.readData(from: readPath, at: revision, offset: offset, length: length) + let read = try await file.readRange(offset, length) let data = Data(read.prefix(length)) var leadingContext = Data() if offset > 0, let first = data.first, (0x80...0xBF).contains(first) { let count = Int(min(offset, 3)) - leadingContext = try await workspace.readData( - from: readPath, at: revision, offset: offset - UInt64(count), length: count - ) + leadingContext = try await file.readRange(offset - UInt64(count), count) } let (content, end) = try toolTextPage(data, path: request.path, hasMore: offset < totalBytes && UInt64(data.count) < totalBytes - offset, @@ -194,30 +216,40 @@ public actor WorkspaceToolAdapter { nextOffset: nextOffset, totalBytes: totalBytes, truncated: nextOffset < totalBytes)) case "workspace_glob": let request = try decoder.decode(GlobArguments.self, from: arguments) - return try encoder.encode( - try await workspace.glob( - request.pattern, - currentDirectory: request.currentDirectory ?? .root, - at: request.revision() - ) - ) + let budget = try toolResultBudget(request.maxBytes) + let offset = try boundedToolInteger(request.matchOffset, default: 0, minimum: 0, maximum: Int.max, name: "match_offset") + let count = try boundedToolInteger(request.maxMatches, default: Self.defaultGlobMaxMatches, minimum: 1, + maximum: Self.maximumGlobMaxMatches, name: "max_matches") + let matches = try await workspace.glob(request.pattern, currentDirectory: request.currentDirectory ?? .root, + at: request.revision()).sorted() + return try boundedGlobOutput(matches, offset: offset, maxMatches: count, maxBytes: budget, encoder: encoder) case "workspace_search": let request = try decoder.decode(SearchArguments.self, from: arguments) return try encoder.encode(try await workspace.search(request.request())) case "workspace_apply": let request = try decoder.decode(ApplyArguments.self, from: arguments) - return try encoder.encode(try await workspace.apply(try request.edits.map { try $0.edit })) + let budget = try toolResultBudget(request.maxBytes) + let fileBudget = try toolFileBudget(request.maxFileBytes) + let result = try await workspace.apply(try request.edits.map { try $0.edit }) + return boundedApplyOutput(result.changes, maxBytes: budget, maxFileBytes: fileBudget, encoder: encoder) case "workspace_diff": let request = try decoder.decode(DiffArguments.self, from: arguments) - let changes = try await workspace.diff( + let budget = try toolResultBudget(request.maxBytes) + let fileBudget = try toolFileBudget(request.maxFileBytes) + let offset = try boundedToolInteger(request.changeOffset, default: 0, minimum: 0, maximum: Int.max, name: "change_offset") + var changes = try await workspace.diff( from: try request.revision(request.fromCheckpointID), to: try request.revision(request.toCheckpointID), options: .init(maxTextBytes: try boundedToolInteger(request.maxTextBytes, default: Self.defaultDiffMaxBytes, minimum: 0, maximum: Self.defaultDiffMaxBytes, name: "max_text_bytes")) ) - return try encoder.encode( - DiffOutput(changes: changes, unifiedPatch: request.unifiedPatch == true ? changes.unifiedPatch() : nil) - ) + if let paths = request.paths { + let selected = Set(paths) + changes = ChangeSet(changes: changes.changes.filter { selected.contains($0.path) }, + textDiffOmissions: changes.textDiffOmissions.filter { selected.contains($0.path) }) + } + return try boundedChangesOutput(changes, offset: offset, maxBytes: budget, maxFileBytes: fileBudget, + patch: request.unifiedPatch == true, applying: false, encoder: encoder) case "workspace_checkpoint": let request = try decoder.decode(CheckpointArguments.self, from: arguments) return try encoder.encode(try await workspace.createCheckpoint(label: request.label)) @@ -258,7 +290,13 @@ private struct GlobArguments: Decodable { var pattern: String var currentDirectory: WorkspacePath? var checkpointID: String? + var matchOffset: Int? + var maxMatches: Int? + var maxBytes: Int? enum CodingKeys: String, CodingKey { + case matchOffset = "match_offset" + case maxMatches = "max_matches" + case maxBytes = "max_bytes" case pattern case currentDirectory = "current_directory" case checkpointID = "checkpoint_id" @@ -315,7 +353,16 @@ private struct SearchArguments: Decodable { } } -private struct ApplyArguments: Decodable { var edits: [ToolEdit] } +private struct ApplyArguments: Decodable { + var edits: [ToolEdit] + var maxBytes: Int? + var maxFileBytes: Int? + enum CodingKeys: String, CodingKey { + case edits + case maxBytes = "max_bytes" + case maxFileBytes = "max_file_bytes" + } +} private struct ToolEdit: Decodable { var kind: String @@ -344,7 +391,15 @@ private struct DiffArguments: Decodable { var toCheckpointID: String? var unifiedPatch: Bool? var maxTextBytes: Int? + var maxBytes: Int? + var maxFileBytes: Int? + var changeOffset: Int? + var paths: [WorkspacePath]? enum CodingKeys: String, CodingKey { + case paths + case maxBytes = "max_bytes" + case maxFileBytes = "max_file_bytes" + case changeOffset = "change_offset" case fromCheckpointID = "from_checkpoint_id" case toCheckpointID = "to_checkpoint_id" case unifiedPatch = "unified_patch" @@ -353,7 +408,6 @@ private struct DiffArguments: Decodable { func revision(_ value: String?) throws -> Revision { try toolRevision(value) } } -private struct DiffOutput: Encodable { var changes: ChangeSet; var unifiedPatch: String? } private struct CheckpointArguments: Decodable { var label: String? } private func toolRevision(_ value: String?) throws -> Revision { @@ -448,3 +502,14 @@ private func toolTextPage( } throw WorkspaceError.invalidEncoding(path) } + +private func toolResultBudget(_ value: Int?) throws -> Int { + try boundedToolInteger(value, default: WorkspaceToolAdapter.defaultResultMaxBytes, + minimum: WorkspaceToolAdapter.minimumResultMaxBytes, + maximum: WorkspaceToolAdapter.maximumResultMaxBytes, name: "max_bytes") +} + +private func toolFileBudget(_ value: Int?) throws -> Int { + try boundedToolInteger(value, default: WorkspaceToolAdapter.defaultFileDetailMaxBytes, minimum: 0, + maximum: WorkspaceToolAdapter.maximumFileDetailMaxBytes, name: "max_file_bytes") +} diff --git a/Sources/Workspace/WorkspaceToolBudget.swift b/Sources/Workspace/WorkspaceToolBudget.swift new file mode 100644 index 0000000..fc063d2 --- /dev/null +++ b/Sources/Workspace/WorkspaceToolBudget.swift @@ -0,0 +1,157 @@ +import Foundation + +/// Adapter envelopes retain the original ChangeSet/EditResult keys. Statistics +/// cover the whole selected operation; the nested ChangeSet describes this page. +struct ToolChangesOutput: Encodable { + var changes = ChangeSet() + var statistics: ChangeSet.Statistics + var totalChanges: Int + var changeOffset: Int + var nextChangeOffset: Int? + var omittedFiles: Int + var omittedTextDiffs = 0 + var truncated = false + var unifiedPatch: String? + // Apply runs atomically: failures throw before this success envelope is built, + // so successful apply responses always carry []; diff omits this field. + var failures: [EditFailure]? +} + +/// This boundary cannot throw after an atomic apply has committed. Even an +/// unexpected encoder error or accounting failure returns a successful, truncated +/// envelope, so a caller never mistakes omitted detail for a failed operation. +func boundedApplyOutput( + _ source: ChangeSet, maxBytes: Int, maxFileBytes: Int, encoder: JSONEncoder +) -> Data { + do { + return try boundedChangesOutput(source, offset: 0, maxBytes: maxBytes, maxFileBytes: maxFileBytes, + patch: false, applying: true, encoder: encoder) + } catch { + return minimalApplyOutput(source) + } +} + +private func minimalApplyOutput(_ source: ChangeSet) -> Data { + let statistics = source.statistics + // Integer interpolation needs no escaping and cannot fail. Even Int.max in + // every field keeps this fixed shape below the 1,024-byte minimum budget. + return Data(""" + {"changes":{"changes":[],"textDiffOmissions":[]},"failures":[],"statistics":{"changedPathCount":\(statistics.changedPathCount),"changedFileCount":\(statistics.changedFileCount),"additions":\(statistics.additions),"deletions":\(statistics.deletions)},"totalChanges":\(source.changes.count),"changeOffset":0,"omittedFiles":\(source.changes.count),"omittedTextDiffs":0,"truncated":true} + """.utf8) +} + +func boundedChangesOutput( + _ source: ChangeSet, offset: Int, maxBytes: Int, maxFileBytes: Int, + patch: Bool, applying: Bool, encoder: JSONEncoder +) throws -> Data { + let start = min(offset, source.changes.count) + var output = ToolChangesOutput(statistics: source.statistics, totalChanges: source.changes.count, + changeOffset: offset, omittedFiles: source.changes.count - start, + unifiedPatch: patch ? "" : nil, failures: applying ? [] : nil) + let largestEnvelope = ToolChangesOutput( + statistics: .init(changedPathCount: Int.max, changedFileCount: Int.max, additions: Int.max, deletions: Int.max), + totalChanges: Int.max, changeOffset: Int.max, nextChangeOffset: applying ? nil : Int.max, + omittedFiles: Int.max, omittedTextDiffs: Int.max, truncated: false, + unifiedPatch: patch ? "" : nil, failures: applying ? [] : nil) + var remaining = maxBytes - (try encoder.encode(largestEnvelope).count) + let omissions = Dictionary(grouping: source.textDiffOmissions, by: \.path) + var patchSections: [String] = [] + var cursor = start + for original in source.changes[start...] { + var change = original + change.diff = nil + // Encode metadata and diff once each. Adding the optional diff property + // costs its JSON key plus its value; no growing prefix is re-encoded. + let metadataCost = try encoder.encode(change).count + 1 // Array separator. + var diffCost = 0 + var section: String? + var sectionCost = 0 + if let diff = original.diff { + diffCost = try encoder.encode(diff).count + if patch { + section = ChangeSet(changes: [original]).unifiedPatch() + // The envelope owns the outer JSON quotes; separators between + // patch sections are encoded as two bytes (backslash and n). + sectionCost = try encoder.encode(section!).count - 2 + (patchSections.isEmpty ? 0 : 2) + } + } + var fileOmissions = omissions[change.path] ?? [] + var omissionCost = 0 + for omission in fileOmissions { + omissionCost += try encoder.encode(omission).count + 1 + } + let textCost = diffCost + sectionCost + let diffPropertyCost = original.diff == nil ? 0 : ",\"diff\":".utf8.count + diffCost + var cost = metadataCost + omissionCost + diffPropertyCost + sectionCost + var omittedText = false + if original.diff != nil, textCost > maxFileBytes || cost > remaining { + omittedText = true + fileOmissions.append(.init(path: change.path, reason: .outputBudgetExceeded)) + omissionCost += try encoder.encode(fileOmissions.last!).count + 1 + section = nil + cost = metadataCost + omissionCost + } else { + change.diff = original.diff + } + guard cost <= remaining else { + if !output.changes.changes.isEmpty { break } + // Oversized path metadata is counted as omitted, but consumes its + // offset so paging cannot get stuck on a single unreturnable entry. + cursor += 1 + continue + } + remaining -= cost + cursor += 1 + output.changes.changes.append(change) + output.changes.textDiffOmissions += fileOmissions + output.omittedTextDiffs += omittedText ? 1 : 0 + if let section { patchSections.append(section) } + } + output.omittedFiles = source.changes.count - start - output.changes.changes.count + output.nextChangeOffset = !applying && cursor < source.changes.count ? cursor : nil + output.truncated = output.omittedFiles > 0 || output.omittedTextDiffs > 0 || + output.changes.textDiffOmissions.contains { $0.reason != .binary } + if patch { output.unifiedPatch = patchSections.joined(separator: "\n") } + let data = try encoder.encode(output) + guard data.count <= maxBytes else { + throw WorkspaceToolError.invalidArguments("max_bytes cannot fit result metadata") + } + return data +} + +struct ToolGlobOutput: Encodable { + var matches: [WorkspacePath] = [] + var totalMatches: Int + var matchOffset: Int + var nextMatchOffset: Int? + var omittedMatches = 0 + var truncated = false +} + +func boundedGlobOutput( + _ matches: [WorkspacePath], offset: Int, maxMatches: Int, maxBytes: Int, encoder: JSONEncoder +) throws -> Data { + var cursor = min(offset, matches.count) + var output = ToolGlobOutput(totalMatches: matches.count, matchOffset: offset) + let largestEnvelope = ToolGlobOutput(totalMatches: Int.max, matchOffset: Int.max, + nextMatchOffset: Int.max, omittedMatches: Int.max, truncated: false) + var remaining = maxBytes - (try encoder.encode(largestEnvelope).count) + while cursor < matches.count, output.matches.count < maxMatches { + let cost = try encoder.encode(matches[cursor]).count + 1 + if cost > remaining { + if !output.matches.isEmpty { break } + output.omittedMatches += 1 + } else { + remaining -= cost + output.matches.append(matches[cursor]) + } + cursor += 1 + } + output.nextMatchOffset = cursor < matches.count ? cursor : nil + output.truncated = cursor < matches.count || output.omittedMatches > 0 + let data = try encoder.encode(output) + guard data.count <= maxBytes else { + throw WorkspaceToolError.invalidArguments("max_bytes cannot fit result metadata") + } + return data +} diff --git a/Tests/WorkspaceTests/FollowupPerformanceTests.swift b/Tests/WorkspaceTests/FollowupPerformanceTests.swift new file mode 100644 index 0000000..fafc0fa --- /dev/null +++ b/Tests/WorkspaceTests/FollowupPerformanceTests.swift @@ -0,0 +1,160 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Bounded follow-up work") +struct FollowupPerformanceTests { + @Test + func `large line diffs have bounded search work and complete statistics`() throws { + let count = 10_000 + let content = (0.. 0) + #expect(work.lines.steps <= TextDiff.maximumSearchSteps) + #expect(work.lines.usedCoarseFallback) + #expect(work.maximumIntralineSteps <= TextDiff.maximumSearchSteps) + #expect(replaced.statistics == .init(originalLineCount: count, updatedLineCount: count, additions: count - 2, deletions: count - 2)) + let hunk = try #require(replaced.hunks.first) + #expect(replaced.hunks.count == 1) + let reconstructed = hunk.lines.filter { $0.kind != .removed }.map { + $0.text + ($0.hasTrailingNewline ? "\n" : "") + }.joined() + #expect(reconstructed == updated) + work = TextDiff.Work() + let longLine = TextDiff.lineBased(from: String(repeating: "a", count: count), + to: String(repeating: "b", count: count), work: &work) + #expect(work.intralineFallbacks == 1) + #expect(work.maximumIntralineSteps <= TextDiff.maximumSearchSteps) + #expect(longLine.hunks.first?.lines.first?.intralineRanges == [.init(location: 0, length: count)]) + work = TextDiff.Work() + let identical = TextDiff.lineBased(from: content, to: content, work: &work) + #expect(work.lines.steps == 0) + #expect(identical.hunks.isEmpty) + } + + @Test + func `checkpoint paging builds its immutable revision index exactly once`() async throws { + struct Page: Decodable { var content: String; var nextOffset: UInt64; var truncated: Bool } + let workspace = Workspace(recording: .off) + let checkpointStore = await workspace.store + let store = try #require(checkpointStore as? InMemoryCheckpointStore) + let content = String(repeating: "x\n", count: 4_096) + try await workspace.writeText("/pages", content) + try await workspace.writeData(Data(repeating: 0, count: 32_000), to: "/unrelated") + let checkpoint = try await workspace.createCheckpoint() + #expect(await store.revisionIndexBuildCount == 0) + let adapter = WorkspaceToolAdapter(workspace: workspace) + var offset: UInt64 = 0 + var reconstructed = "" + var pages = 0 + repeat { + let args = "{\"path\":\"/pages\",\"checkpoint_id\":\"\(checkpoint.id)\",\"offset\":\(offset),\"max_bytes\":128}" + let page = try JSONDecoder().decode(Page.self, from: await adapter.call(name: "workspace_read", arguments: Data(args.utf8))) + #expect(page.nextOffset > offset) + reconstructed += page.content + offset = page.nextOffset + pages += 1 + if !page.truncated { break } + } while true + #expect(pages == 64) + #expect(await store.revisionIndexBuildCount == 1) + #expect(reconstructed == content) + try await workspace.writeText("/pages", "new current content") + #expect(try await workspace.readText("/pages", at: .checkpoint(checkpoint.id)) == content) + #expect(await store.revisionIndexBuildCount == 1) + } + + @Test + func `paging thousands of changes and glob matches encodes each entry once`() throws { + struct DiffPage: Decodable { var changes: ChangeSet; var nextChangeOffset: Int?; var unifiedPatch: String? } + struct GlobPage: Decodable { var matches: [WorkspacePath]; var nextMatchOffset: Int? } + let diff = TextDiff.lineBased(from: "old\n", to: "new\n") + let source = ChangeSet(changes: (0..<3_000).map { + .init(path: WorkspacePath(normalizing: "/file-\($0)"), kind: .file, effect: .modified, diff: diff) + }) + let expected = source.changes.map(\.path) + var offset = 0 + var collected: [WorkspacePath] = [] + repeat { + let encoder = CountingOutputEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try boundedChangesOutput(source, offset: offset, maxBytes: 262_144, maxFileBytes: 8_192, + patch: true, applying: false, encoder: encoder) + #expect(data.count <= 262_144) + let page = try JSONDecoder().decode(DiffPage.self, from: data) + #expect(encoder.changeEncodes >= page.changes.changes.count) + #expect(encoder.diffEncodes == encoder.changeEncodes) + #expect(encoder.patchEncodes == encoder.changeEncodes) + #expect(encoder.changeEncodes <= page.changes.changes.count + 1) + #expect(encoder.envelopeEncodes == 1) + #expect(encoder.finalEncodes == 1) + #expect(encoder.totalEncodes <= encoder.changeEncodes * 4 + 2) + #expect(page.unifiedPatch == page.changes.unifiedPatch()) + collected += page.changes.changes.map(\.path) + guard let next = page.nextChangeOffset else { break } + #expect(next > offset) + offset = next + } while true + #expect(collected == expected) + offset = 0 + collected = [] + repeat { + let encoder = CountingOutputEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try boundedGlobOutput(expected, offset: offset, maxMatches: 500, maxBytes: 65_536, + encoder: encoder) + #expect(data.count <= 65_536) + let page = try JSONDecoder().decode(GlobPage.self, from: data) + #expect(encoder.matchEncodes >= page.matches.count) + #expect(encoder.matchEncodes <= page.matches.count + 1) + #expect(encoder.envelopeEncodes == 1) + #expect(encoder.finalEncodes == 1) + #expect(encoder.totalEncodes == encoder.matchEncodes + 2) + collected += page.matches + guard let next = page.nextMatchOffset else { break } + #expect(next > offset) + offset = next + } while true + #expect(collected == expected) + } + /// Count actual top-level encoder calls, including any future accidental + /// re-encoding of a growing envelope, without instrumentation in production. + private final class CountingOutputEncoder: JSONEncoder, @unchecked Sendable { + var changeEncodes = 0 + var diffEncodes = 0 + var patchEncodes = 0 + var matchEncodes = 0 + var envelopeEncodes = 0 + var finalEncodes = 0 + var totalEncodes = 0 + override func encode(_ value: T) throws -> Data { + totalEncodes += 1 + if value is FileChange { changeEncodes += 1 } + if value is TextDiff { diffEncodes += 1 } + if value is String { patchEncodes += 1 } + if value is WorkspacePath { matchEncodes += 1 } + if let output = value as? ToolChangesOutput { + if output.totalChanges == Int.max { envelopeEncodes += 1 } else { finalEncodes += 1 } + } + if let output = value as? ToolGlobOutput { + if output.totalMatches == Int.max { envelopeEncodes += 1 } else { finalEncodes += 1 } + } + return try super.encode(value) + } + } + +} diff --git a/Tests/WorkspaceTests/TextDiffTests.swift b/Tests/WorkspaceTests/TextDiffTests.swift index 9dedff4..a5aa383 100644 --- a/Tests/WorkspaceTests/TextDiffTests.swift +++ b/Tests/WorkspaceTests/TextDiffTests.swift @@ -126,4 +126,92 @@ struct TextDiffTests { #expect(hunk.lines.filter { $0.kind == .added }.allSatisfy { $0.hasTrailingNewline }) #expect(TextDiff.lineBased(from: "\r\n\r\n", to: "\r\n\r\n").originalLineCount == 2) } + @Test(arguments: [1_500, 20_000]) + func `distant sparse edits retain separate precise hunks`(count: Int) { + let original = (1...count).map { "line \($0)" } + var updated = original + updated[9] = "changed ten" + updated[count - 11] = "changed near end" + let diff = TextDiff.lineBased(from: original.joined(separator: "\n"), to: updated.joined(separator: "\n")) + #expect(diff.statistics.additions == 2) + #expect(diff.statistics.deletions == 2) + #expect(diff.hunks.count == 2) + } + + @Test + func `a small edit and a pasted function retain minimal separate hunks`() { + let original = (0..<10_000).map { "let value\($0) = \($0)" } + var updated = original + updated[10] = "let value10 = -1" + let function = ["func pastedFunction() {"] + (0..<298).map { " consume(\($0))" } + ["}"] + updated.insert(contentsOf: function, at: 9_000) + var work = TextDiff.Work() + let diff = TextDiff.lineBased(from: original.joined(separator: "\n"), + to: updated.joined(separator: "\n"), work: &work) + #expect(diff.statistics == .init(originalLineCount: 10_000, updatedLineCount: 10_300, additions: 301, deletions: 1)) + #expect(diff.hunks.count == 2) + #expect(!work.lines.usedCoarseFallback) + #expect(work.lines.steps <= TextDiff.maximumSearchSteps) + } + + @Test + func `scattered code edits retain minimal separate hunks`() { + let original = (0..<20_000).map { "let value\($0) = \($0)" } + var updated = original + for index in stride(from: 0, to: 20_000, by: 100) { + // The first of 200 candidate edits is unchanged; 199 are actual edits. + updated[index] = "let value\(index) = \(index * 2)" + } + var work = TextDiff.Work() + let diff = TextDiff.lineBased(from: original.joined(separator: "\n"), + to: updated.joined(separator: "\n"), work: &work) + #expect(diff.statistics == .init(originalLineCount: 20_000, updatedLineCount: 20_000, additions: 199, deletions: 199)) + #expect(diff.hunks.count == 199) + #expect(!work.lines.usedCoarseFallback) + #expect(work.lines.steps <= TextDiff.maximumSearchSteps) + } + + @Test + func `long line pairs retain independent precise character highlights`() throws { + let original = String(repeating: "a", count: 1_000) + "x" + String(repeating: "b", count: 1_000) + let updated = String(repeating: "a", count: 1_000) + "y" + String(repeating: "b", count: 1_000) + let diff = TextDiff.lineBased(from: original + "\n" + original, to: updated + "\n" + updated) + let lines = try #require(diff.hunks.first).lines.filter { $0.kind != .context } + #expect(lines.count == 4) + #expect(lines.allSatisfy { $0.intralineRanges == [.init(location: 1_000, length: 1)] }) + // Each edge-edited 500-character pair used to consume the entire shared + // square-size budget, making the second pair a whole-line highlight. + let first = "a" + String(repeating: "x", count: 498) + "b" + let second = "A" + String(repeating: "y", count: 498) + "B" + let firstUpdated = "c" + String(repeating: "x", count: 498) + "d" + let secondUpdated = "C" + String(repeating: "y", count: 498) + "D" + let edgeDiff = TextDiff.lineBased(from: first + "\n" + second, to: firstUpdated + "\n" + secondUpdated) + let edgeLines = try #require(edgeDiff.hunks.first).lines.filter { $0.kind != .context } + #expect(edgeLines.count == 4) + #expect(edgeLines.allSatisfy { + $0.intralineRanges == [.init(location: 0, length: 1), .init(location: 499, length: 1)] + }) + + } + + @Test(arguments: [ + ("a\nb\nc\n", "a\nx\nb\nc\n", 1, 0), + ("a\nb\nc\n", "a\nc\n", 0, 1), + ("a\nb\na\n", "b\na\nb\n", 1, 1), + ("a\na\nb\n", "a\nb\nb\n", 1, 1), + ("a\nb\nc\n", "c\nb\na\n", 2, 2), + ("a\na\na\nb\n", "b\na\na\na\n", 1, 1), + ("a\nb\nc\n", "x", 1, 3), + ]) + func `shortest scripts preserve both sides through uneven and repeated lines`( + original: String, updated: String, additions: Int, deletions: Int + ) { + let diff = TextDiff.lineBased(from: original, to: updated) + #expect(diff.statistics.additions == additions) + #expect(diff.statistics.deletions == deletions) + let lines = diff.hunks.flatMap(\.lines) + #expect(lines.filter { $0.kind != .added }.map { $0.text + ($0.hasTrailingNewline ? "\n" : "") }.joined() == original) + #expect(lines.filter { $0.kind != .removed }.map { $0.text + ($0.hasTrailingNewline ? "\n" : "") }.joined() == updated) + } + } diff --git a/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift b/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift index 4b91553..d1dbd1a 100644 --- a/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift +++ b/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift @@ -96,6 +96,14 @@ struct WorkspaceToolAdapterTests { ("workspace_search", #"{"pattern":"TODO","context_lines":9223372036854775808}"#), ("workspace_search", #"{"pattern":"TODO","match_offset":-1}"#), ("workspace_search", #"{"pattern":"TODO","max_bytes":-1}"#), + ("workspace_glob", #"{"pattern":"*","match_offset":-1}"#), + ("workspace_glob", #"{"pattern":"*","max_matches":0}"#), + ("workspace_glob", #"{"pattern":"*","max_matches":1.5}"#), + ("workspace_glob", #"{"pattern":"*","max_bytes":1023}"#), + ("workspace_apply", #"{"edits":[],"max_bytes":-1}"#), + ("workspace_apply", #"{"edits":[],"max_file_bytes":-1}"#), + ("workspace_diff", #"{"change_offset":-1}"#), + ("workspace_diff", #"{"max_file_bytes":1.5}"#), ("workspace_diff", #"{"max_text_bytes":-1}"#), ("workspace_diff", #"{"max_text_bytes":1.5}"#), ("workspace_read", #"{"path":"/note","offset":-1}"#), @@ -112,7 +120,6 @@ struct WorkspaceToolAdapterTests { @Test func `read pages are bounded lossless UTF8 and report total bytes`() async throws { let workspace = Workspace(recording: .off) - // Keep the line count small to avoid the existing quadratic checkpoint diff cost. let content = "abc" + String(repeating: String(repeating: "abc😀", count: 999) + "\n", count: 52) try await workspace.writeText("/note", content) let checkpoint = try await workspace.createCheckpoint() @@ -254,26 +261,48 @@ struct WorkspaceToolAdapterTests { } } - @Test - func `read resolves intermediate symlinks before measuring and reading content`() async throws { - let workspace = Workspace(recording: .off) + @Test(arguments: [false, true]) + func `read resolves intermediate symlinks before measuring and reading content`(diskBacked: Bool) async throws { + let root = try TestSupport.temporaryDirectory("RevisionSymlinks") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: diskBacked ? .directory(root) : .memory, recording: .off) try await workspace.createDirectory("/x") try await workspace.createDirectory("/y/real") try await workspace.writeText("/x/note", "SHORT") try await workspace.writeText("/y/note", "a much longer file body here") try await workspace.createSymbolicLink("/x/a", target: "/y/real") try await workspace.createSymbolicLink("/y/real/link", target: "../note") + try await workspace.createSymbolicLink("/x/out", target: "../../../../y") + try await workspace.createSymbolicLink("/loop-a", target: "/loop-b") + try await workspace.createSymbolicLink("/loop-b", target: "/loop-a") + try await workspace.createSymbolicLink("/dangling", target: "/absent") let expected = try await workspace.readText("/x/a/link") #expect(expected == "a much longer file body here") + let checkpoint = try await workspace.createCheckpoint() let adapter = WorkspaceToolAdapter(workspace: workspace) - let data = try await adapter.call(name: "workspace_read", - arguments: Data(#"{"path":"/x/a/link"}"#.utf8)) - let page = try JSONDecoder().decode(ReadPage.self, from: data) - #expect(page.content == expected) - #expect(page.totalBytes == UInt64(expected.utf8.count)) - #expect(page.nextOffset == UInt64(expected.utf8.count)) - #expect(!page.truncated) + for (path, code) in [("/loop-a/note", ELOOP), ("/dangling/note", ENOENT), + ("/x/note/child", ENOTDIR), ("/x/a", EISDIR)] { + do { + _ = try await workspace.readData(from: WorkspacePath(path), at: .checkpoint(checkpoint.id)) + Issue.record("expected POSIX error for \(path)") + } catch { + #expect((error as NSError).domain == NSPOSIXErrorDomain) + #expect((error as NSError).code == Int(code)) + } + } + #expect(try await workspace.readText("/x/out/note", at: .checkpoint(checkpoint.id)) == expected) + for revision in [Revision.current, .checkpoint(checkpoint.id)] { + #expect(try await workspace.readText("/x/a/link", at: revision) == expected) + let suffix = revision == .current ? "" : ",\"checkpoint_id\":\"\(checkpoint.id)\"" + let data = try await adapter.call(name: "workspace_read", + arguments: Data("{\"path\":\"/x/a/link\"\(suffix)}".utf8)) + let page = try JSONDecoder().decode(ReadPage.self, from: data) + #expect(page.content == expected) + #expect(page.totalBytes == UInt64(expected.utf8.count)) + #expect(page.nextOffset == UInt64(expected.utf8.count)) + #expect(!page.truncated) + } } @Test @@ -301,6 +330,217 @@ struct WorkspaceToolAdapterTests { #expect(try JSONDecoder().decode(SearchResult.self, from: beyond).matches.isEmpty) } + @Test + func `glob bounds escaped paths and pages without gaps`() async throws { + struct Page: Decodable { + var matches: [WorkspacePath] + var totalMatches: Int + var truncated: Bool + var nextMatchOffset: Int? + var omittedMatches: Int + } + let workspace = Workspace(recording: .off) + for index in 0..<40 { + let path = try WorkspacePath("/file-\(index)-" + String(repeating: "\u{1}", count: 80)) + try await workspace.writeText(path, "value") + } + let expected = try await workspace.glob("/file-*") + let adapter = WorkspaceToolAdapter(workspace: workspace) + var offset = 0 + var collected: [WorkspacePath] = [] + repeat { + let args = "{\"pattern\":\"/file-*\",\"max_bytes\":1024,\"max_matches\":3,\"match_offset\":\(offset)}" + let data = try await adapter.call(name: "workspace_glob", arguments: Data(args.utf8)) + #expect(data.count <= 1024) + let page = try JSONDecoder().decode(Page.self, from: data) + #expect(page.totalMatches == 40) + #expect(page.matches.count <= 3) + #expect(!page.matches.isEmpty) + #expect(page.omittedMatches == 0) + collected += page.matches + guard let next = page.nextMatchOffset else { break } + #expect(page.truncated) + #expect(next > offset) + offset = next + } while true + #expect(collected == expected) + let countLimited = try await adapter.call(name: "workspace_glob", arguments: Data(#"{"pattern":"/file-*","max_matches":3,"max_bytes":9223372036854775807}"#.utf8)) + let countPage = try JSONDecoder().decode(Page.self, from: countLimited) + #expect(countPage.matches.count == 3) + #expect(countPage.nextMatchOffset == 3) + #expect(countPage.truncated) + let beyond = try await adapter.call(name: "workspace_glob", arguments: Data(#"{"pattern":"*","match_offset":9223372036854775807,"max_bytes":9223372036854775807}"#.utf8)) + let empty = try JSONDecoder().decode(Page.self, from: beyond) + #expect(empty.matches.isEmpty) + #expect(!empty.truncated) + } + + @Test + func `diff pages bound text patch and metadata while apply preserves the full effect`() async throws { + struct Page: Decodable { + var changes: ChangeSet + var statistics: ChangeSet.Statistics + var totalChanges: Int + var omittedFiles: Int + var omittedTextDiffs: Int + var truncated: Bool + var nextChangeOffset: Int? + var unifiedPatch: String? + var failures: [EditFailure]? + } + let workspace = Workspace(recording: .off) + let checkpoint = try await workspace.createCheckpoint() + let adapter = WorkspaceToolAdapter(workspace: workspace) + let text = String(repeating: "changed\u{1}😀\n", count: 100) + var edits: [JSONValue] = [] + for index in 0..<40 { + edits.append(.object(["kind": .string("write_text"), "path": .string("/file-\(index)"), "content": .string(text)])) + } + let applied = try await adapter.call(name: "workspace_apply", arguments: JSONEncoder().encode([ + "edits": JSONValue.array(edits), "max_bytes": .number(1024), "max_file_bytes": .number(512), + ])) + #expect(applied.count <= 1024) + let result = try JSONDecoder().decode(Page.self, from: applied) + #expect(result.totalChanges == 40) + #expect(result.statistics.changedFileCount == 40) + #expect(result.statistics.additions == 4000) + #expect(result.truncated) + #expect(result.omittedFiles > 0) + #expect(result.omittedTextDiffs > 0) + #expect(result.failures?.isEmpty == true) + for index in 0..<40 { + #expect(try await workspace.readText(WorkspacePath("/file-\(index)")) == text) + } + var offset = 0 + var paths: [WorkspacePath] = [] + repeat { + let args: [String: JSONValue] = ["from_checkpoint_id": .string(checkpoint.id.uuidString), + "unified_patch": .boolean(true), "max_bytes": .number(1024), "max_file_bytes": .number(512), + "change_offset": .number(Double(offset))] + let data = try await adapter.call(name: "workspace_diff", arguments: JSONEncoder().encode(args)) + #expect(data.count <= 1024) + let page = try JSONDecoder().decode(Page.self, from: data) + #expect(page.totalChanges == 40) + #expect(page.statistics.additions == 4000) + #expect(page.truncated) + #expect(page.omittedTextDiffs == page.changes.changes.count) + #expect(page.unifiedPatch == "") + #expect(page.changes.textDiffOmissions.allSatisfy { $0.reason.rawValue == "outputBudgetExceeded" }) + paths += page.changes.changes.map(\.path) + guard let next = page.nextChangeOffset else { break } + #expect(next > offset) + offset = next + } while true + #expect(paths == (0..<40).map { WorkspacePath(normalizing: "/file-\($0)") }.sorted()) + let narrowed = try await adapter.call(name: "workspace_diff", arguments: JSONEncoder().encode([ + "from_checkpoint_id": JSONValue.string(checkpoint.id.uuidString), "paths": .array([.string("/file-0")]), + "max_file_bytes": .number(65_536), "unified_patch": .boolean(true), + ])) + #expect(narrowed.count <= 65_536) + let page = try JSONDecoder().decode(Page.self, from: narrowed) + #expect(page.totalChanges == 1) + #expect(page.changes.changes.first?.diff?.statistics.additions == 100) + #expect(page.unifiedPatch?.contains("+changed") == true) + #expect(!page.truncated) + } + + @Test + func `diff default and maximum budgets include escaped text and patch bytes`() async throws { + struct Page: Decodable { + var changes: ChangeSet + var statistics: ChangeSet.Statistics + var truncated: Bool + var omittedTextDiffs: Int + var unifiedPatch: String? + } + let workspace = Workspace(recording: .off) + let before = try await workspace.createCheckpoint() + let text = String(repeating: "\u{1}😀", count: 20_000) + try await workspace.writeText("/large", text) + let adapter = WorkspaceToolAdapter(workspace: workspace) + for extra in ["", ",\"max_bytes\":9223372036854775807,\"max_file_bytes\":9223372036854775807"] { + let args = "{\"from_checkpoint_id\":\"\(before.id)\",\"unified_patch\":true\(extra)}" + let data = try await adapter.call(name: "workspace_diff", arguments: Data(args.utf8)) + #expect(data.count <= (extra.isEmpty ? 65_536 : 262_144)) + let page = try JSONDecoder().decode(Page.self, from: data) + #expect(page.truncated) + #expect(page.omittedTextDiffs == 1) + #expect(page.changes.changes.first?.diff == nil) + #expect(page.statistics.additions == 1) + #expect(page.unifiedPatch == "") + } + let applied = try await adapter.call(name: "workspace_apply", arguments: JSONEncoder().encode([ + "edits": JSONValue.array([.object(["kind": .string("write_text"), "path": .string("/default"), "content": .string(text)])]), + ])) + #expect(applied.count <= 65_536) + let page = try JSONDecoder().decode(Page.self, from: applied) + #expect(page.truncated) + #expect(page.omittedTextDiffs == 1) + #expect(try await workspace.readText("/default") == text) + } + + @Test + func `oversized path metadata is reported and pagination makes progress`() async throws { + struct GlobPage: Decodable { var matches: [WorkspacePath]; var omittedMatches: Int; var truncated: Bool } + struct DiffPage: Decodable { var changes: ChangeSet; var omittedFiles: Int; var truncated: Bool } + let workspace = Workspace(recording: .off) + let checkpoint = try await workspace.createCheckpoint() + try await workspace.writeText(WorkspacePath("/a" + String(repeating: "x", count: 2000)), "value") + try await workspace.writeText("/z", "value") + let adapter = WorkspaceToolAdapter(workspace: workspace) + let glob = try await adapter.call(name: "workspace_glob", arguments: Data(#"{"pattern":"/[az]*","max_bytes":1024}"#.utf8)) + #expect(glob.count <= 1024) + let globPage = try JSONDecoder().decode(GlobPage.self, from: glob) + #expect(globPage.matches == ["/z"]) + #expect(globPage.omittedMatches == 1) + #expect(globPage.truncated) + let diff = try await adapter.call(name: "workspace_diff", arguments: Data("{\"from_checkpoint_id\":\"\(checkpoint.id)\",\"max_bytes\":1024}".utf8)) + #expect(diff.count <= 1024) + let diffPage = try JSONDecoder().decode(DiffPage.self, from: diff) + #expect(diffPage.changes.touchedPaths == ["/z"]) + #expect(diffPage.omittedFiles == 1) + #expect(diffPage.truncated) + } + + @Test(arguments: [false, true]) + func `committed apply returns a minimal success envelope when detail encoding fails`(encoderThrows: Bool) async throws { + struct Page: Decodable { + var changes: ChangeSet + var statistics: ChangeSet.Statistics + var totalChanges: Int + var failures: [EditFailure] + var truncated: Bool + } + let workspace = Workspace(recording: .off) + let result = try await workspace.apply([.appendText("/one", "first"), .writeText("/two", "second")]) + let data = boundedApplyOutput(result.changes, maxBytes: 1024, maxFileBytes: 8192, + encoder: ResponseFaultEncoder(encoderThrows: encoderThrows)) + #expect(data.count <= 1024) + let page = try JSONDecoder().decode(Page.self, from: data) + #expect(page.changes.isEmpty) + #expect(page.statistics.changedFileCount == 2) + #expect(page.totalChanges == 2) + #expect(page.failures.isEmpty) + #expect(page.truncated) + #expect(try await workspace.readText("/one") == "first") + #expect(try await workspace.readText("/two") == "second") + } + + /// Exercise both post-commit failure routes without a test-only adapter flag. + private final class ResponseFaultEncoder: JSONEncoder, @unchecked Sendable { + let encoderThrows: Bool + init(encoderThrows: Bool) { self.encoderThrows = encoderThrows; super.init() } + override func encode(_ value: T) throws -> Data { + if encoderThrows { throw WorkspaceToolError.invalidArguments("injected encoding failure") } + var data = try super.encode(value) + if let output = value as? ToolChangesOutput, output.totalChanges != Int.max { + // Valid JSON whitespace simulates a final result exceeding the estimate. + data.append(Data(repeating: 0x20, count: 1024)) + } + return data + } + } + private struct ReadPage: Decodable { var content: String var offset: UInt64 From a25ccea8ec4c4b16e38493a810695f993792d1f0 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 00:23:43 -0700 Subject: [PATCH 11/34] Parse path syntax on UTF-8 bytes to close a root-jail escape Workspace parsed POSIX paths with Swift grapheme-cluster operations. A grapheme-extending scalar following `/` (e.g. a combining mark, ZWJ, variation selector or spacing mark) could hide that separator from component splitting and containment checks, while the kernel still treated its UTF-8 byte as a separator. LocalFileSystem could therefore pass several components to one descriptor-relative syscall, so an intermediate symlink escaped the root despite O_NOFOLLOW. Because createSymlink accepts arbitrary targets, a workspace client could read, overwrite, delete or hard-link existing files and create new files and directories outside the root in any directory a symlink could name, subject to host filesystem permissions, provided the name began with such a scalar. The same mismatch let paths bypass authorization rules, mount routing, overlay deletions, descendant-move protection and quotas, and broke checkpoint, archive and revision lookups and recursive event filters. This fix parses path syntax, including globs, on UTF-8 bytes; compares containment component-wise with the same canonical-equivalence rules as WorkspacePath equality; and rejects empty, `.`, `..`, slash- or NUL-containing component names before `*at()` calls, except the explicit `.` reference to an already-resolved directory descriptor. --- Sources/Workspace/CheckpointStore.swift | 4 +- Sources/Workspace/FS/InMemoryFileSystem.swift | 2 +- Sources/Workspace/FS/LimitedFileSystem.swift | 2 +- Sources/Workspace/FS/LocalFileSystem.swift | 58 ++-- Sources/Workspace/FS/MountedFileSystem.swift | 25 +- Sources/Workspace/FS/OverlayFileSystem.swift | 12 +- Sources/Workspace/RevisionIndex.swift | 2 +- Sources/Workspace/Snapshot.swift | 2 +- Sources/Workspace/Support/Permissions.swift | 2 +- Sources/Workspace/Support/WorkspacePath.swift | 115 ++++--- Sources/Workspace/UnifiedPatch.swift | 2 +- Sources/Workspace/WorkspaceArchive.swift | 7 +- Sources/Workspace/WorkspaceEvent.swift | 2 +- Sources/Workspace/WorkspaceRevision.swift | 2 +- .../UnicodePathRegressionTests.swift | 286 ++++++++++++++++++ 15 files changed, 420 insertions(+), 103 deletions(-) create mode 100644 Tests/WorkspaceTests/UnicodePathRegressionTests.swift diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 1134202..6f21b51 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -617,7 +617,7 @@ actor FileCheckpointStore: CheckpointStore { in node: SnapshotManifest.Node ) -> SnapshotManifest.Node? { if node.path == path { return node } - for child in node.children ?? [] where child.path == path || path.string.hasPrefix(child.path.string + "/") { + for child in node.children ?? [] where path.isWithin(child.path) { return manifestNodeStatic(at: path, in: child) } return nil @@ -720,7 +720,7 @@ actor FileCheckpointStore: CheckpointStore { private func manifestNode(at path: WorkspacePath, in node: SnapshotManifest.Node) -> SnapshotManifest.Node? { if node.path == path { return node } - for child in node.children ?? [] where child.path == path || path.string.hasPrefix(child.path.string + "/") { + for child in node.children ?? [] where path.isWithin(child.path) { return manifestNode(at: path, in: child) } return nil diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index c62f94d..c81da9d 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -228,7 +228,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { } if sourceNode.isDirectory, - (destinationPath == sourcePath || destinationPath.string.hasPrefix(sourcePath.string + "/")) { + destinationPath.isWithin(sourcePath) { throw posixError(EINVAL) } diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index 9f283a3..b1f6ef1 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -328,7 +328,7 @@ public actor LimitedFileSystem: FileSystem, PotentialPathResolvingFileSystem { } private static func contains(_ path: WorkspacePath, in parent: WorkspacePath) -> Bool { - parent.isRoot || path == parent || path.string.hasPrefix(parent.string + "/") + path.isWithin(parent) } private static func posix(_ code: Int32) -> NSError { diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index f2f9b3b..d343203 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -58,6 +58,29 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let parent: Descriptor let name: String let path: WorkspacePath + + // Every variable *at() name comes from an immutable, validated Entry. + // Even a future parser bug must not pass a multi-component path to the kernel. + init(parent: Descriptor, name: String, path: WorkspacePath) throws { + try LocalFileSystem.validateEntryName(name) + self.parent = parent + self.name = name + self.path = path + } + + // The only permitted dot entry refers to an already resolved directory fd. + init(directory: Descriptor, path: WorkspacePath) { + self.parent = directory + self.name = "." + self.path = path + } + } + + private static func validateEntryName(_ name: String) throws { + guard !name.isEmpty, name != ".", name != "..", + !name.utf8.contains(47), !name.utf8.contains(0) else { + throw WorkspaceError.invalidPath(name) + } } /// Creates an unconfigured disk-backed filesystem. @@ -155,13 +178,13 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem /// Absolute disk symlink targets are host paths. Accept physical, Foundation-resolved, and /// configured root spellings, mapping each to the retained descriptor with exact boundaries. private func absoluteComponents(_ target: String, configuration: ConfigurationSnapshot) throws -> [String] { + // Keep dot components for descriptor-relative resolution; lexical normalization + // here would erase symlink traversal before a subsequent "..". + let targetComponents = WorkspacePath.splitComponents(target) for root in [configuration.resolvedRootPath, configuration.foundationRootPath, configuration.rootURL.path] { - if target == root { - return [] - } - let prefix = root == "/" ? "/" : root + "/" - if target.hasPrefix(prefix) { - return String(target.dropFirst(prefix.count)).split(separator: "/").map(String.init) + let rootComponents = WorkspacePath.splitComponents(root) + if targetComponents.starts(with: rootComponents) { + return Array(targetComponents.dropFirst(rootComponents.count)) } } throw WorkspaceError.invalidPath(target) @@ -173,7 +196,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem configuration: ConfigurationSnapshot? = nil ) throws -> Entry { let config = try configuration ?? requireConfiguration() - guard !path.string.contains("\u{0}") else { + guard !path.string.utf8.contains(0) else { throw WorkspaceError.invalidPath(path.string) } var pending = path.components @@ -194,7 +217,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem current = config.root missingAncestor = false for ancestor in names { - let entry = Entry(parent: current, name: ancestor, path: .root) + let entry = try Entry(parent: current, name: ancestor, path: .root) do { current = try openEntry(entry, flags: Self.searchDirectoryFlags) } catch let error as NSError where allowMissingAncestors @@ -206,7 +229,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem continue } let final = pending.isEmpty - let entry = Entry( + let entry = try Entry( parent: current, name: name, path: WorkspacePath(unchecked: "/" + (names + [name]).joined(separator: "/"))) if final && !followFinal { @@ -234,7 +257,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem guard createParents else { throw error } - let result = mkdirat(entry.parent.value, name, 0o755) + let result = mkdirat(entry.parent.value, entry.name, 0o755) if result < 0 && errno != EEXIST { try check(result) } @@ -246,12 +269,12 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } let target = try linkTarget(entry) - if target.hasPrefix("/") { + if target.utf8.first == 47 { pending = try absoluteComponents(target, configuration: config) + pending current = config.root names = [] } else { - pending = target.split(separator: "/").map(String.init) + pending + pending = WorkspacePath.splitComponents(target) + pending } continue } @@ -261,7 +284,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem current = try openEntry(entry, flags: Self.searchDirectoryFlags) names.append(name) } - return Entry(parent: current, name: ".", path: WorkspacePath(unchecked: "/" + names.joined(separator: "/"))) + return Entry(directory: current, path: WorkspacePath(unchecked: "/" + names.joined(separator: "/"))) } private func openEntry(_ entry: Entry, flags: Int32, mode: mode_t = 0o666) throws -> Descriptor { @@ -463,13 +486,13 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem private func replace(_ entry: Entry, body: (Descriptor) throws -> Void) throws { let temporary = ".workspace-" + UUID().uuidString - let temporaryEntry = Entry(parent: entry.parent, name: temporary, path: entry.path) + let temporaryEntry = try Entry(parent: entry.parent, name: temporary, path: entry.path) let fd = try openEntry(temporaryEntry, flags: O_WRONLY | O_CREAT | O_EXCL) defer { - _ = unlinkat(entry.parent.value, temporary, 0) + _ = unlinkat(temporaryEntry.parent.value, temporaryEntry.name, 0) } try body(fd) - try check(renameat(entry.parent.value, temporary, entry.parent.value, entry.name)) + try check(renameat(temporaryEntry.parent.value, temporaryEntry.name, entry.parent.value, entry.name)) } /// See ``FileSystem/createDirectory(path:recursive:)``. @@ -626,8 +649,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let config = try requireConfiguration() let source = try resolve(sourcePath, followFinal: false, configuration: config) let destination = try resolve(destinationPath, followFinal: false, createParents: true, configuration: config) - guard source.path != destination.path, - !(destination.path.string.hasPrefix(source.path.string + "/")) && !source.path.isRoot + guard !destination.path.isWithin(source.path) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EINVAL)) } diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index 3ccf068..cdb6388 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -54,7 +54,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst mounts: [Mount] = [] ) { self.base = base - self.mounts = mounts.sorted { $0.mountPoint.string.count > $1.mountPoint.string.count } + self.mounts = mounts.sorted { $0.mountPoint.components.count > $1.mountPoint.components.count } } /// Adds a filesystem mount at `mountPoint`. @@ -62,7 +62,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst withLock { let mount = Mount(mountPoint: mountPoint, fileSystem: fileSystem) mounts.append(mount) - mounts.sort { $0.mountPoint.string.count > $1.mountPoint.string.count } + mounts.sort { $0.mountPoint.components.count > $1.mountPoint.components.count } } } @@ -277,7 +277,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { let resolved = resolveWritable(path: path, mounts: mountsSnapshot()) let real = try await resolved.filesystem.resolveRealPath(path: resolved.relativePath) - return resolved.mountPoint.isRoot ? real : WorkspacePath.join(resolved.mountPoint, String(real.string.dropFirst())) + return resolved.mountPoint.isRoot ? real : WorkspacePath.join(resolved.mountPoint, real.components.joined(separator: "/")) } // Used only when a symlink target or suffix is missing. Resolve inside the selected @@ -287,7 +287,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst let real = try await FileSystemPathResolver(base: resolved.filesystem).resolve( resolved.relativePath, features: await resolved.filesystem.capabilities(), followFinal: true ) - return resolved.mountPoint.isRoot ? real : WorkspacePath.join(resolved.mountPoint, String(real.string.dropFirst())) + return resolved.mountPoint.isRoot ? real : WorkspacePath.join(resolved.mountPoint, real.components.joined(separator: "/")) } /// See ``FileSystem/exists(path:)``. @@ -392,22 +392,15 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst guard isPath(mount.mountPoint, inside: path) else { continue } - let remaining = mount.mountPoint.isRoot - ? "" - : String(mount.mountPoint.string.dropFirst(path.isRoot ? 1 : path.string.count + 1)) - guard !remaining.isEmpty else { continue } - if let first = remaining.split(separator: "/").first { - names.insert(String(first)) + if let first = mount.mountPoint.components.dropFirst(path.components.count).first { + names.insert(first) } } return names.sorted() } private func isPath(_ candidate: WorkspacePath, inside parent: WorkspacePath) -> Bool { - if parent.isRoot { - return candidate.string.hasPrefix("/") && !candidate.isRoot - } - return candidate == parent || candidate.string.hasPrefix(parent.string + "/") + candidate.isWithin(parent) } private func resolveWritable(path: WorkspacePath, mounts: [Mount]) @@ -428,8 +421,8 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst return (mount.mountPoint, mount.fileSystem, path) } - if !mount.mountPoint.isRoot, path.string.hasPrefix(mount.mountPoint.string + "/") { - let suffix = String(path.string.dropFirst(mount.mountPoint.string.count)) + if path.isWithin(mount.mountPoint) { + let suffix = path.components.dropFirst(mount.mountPoint.components.count).joined(separator: "/") return ( mount.mountPoint, mount.fileSystem, diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index 17dd44e..36a16a6 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -308,7 +308,7 @@ public actor OverlayFileSystem: FileSystem { } if lowerVisible { // Deeper cuts are redundant once this path is cut. - cuts = cuts.filter { !$0.string.hasPrefix(path.string + "/") } + cuts = cuts.filter { !$0.isWithin(path) } cuts.insert(path) } } @@ -323,7 +323,7 @@ public actor OverlayFileSystem: FileSystem { guard let sourceInfo = await mergedEntry(sourcePath) else { throw posixError(ENOENT) } - if sourceInfo.kind == .directory, destinationPath.string.hasPrefix(sourcePath.string + "/") { + if sourceInfo.kind == .directory, destinationPath.isWithin(sourcePath) { throw posixError(EINVAL) } guard await mergedEntry(destinationPath) == nil else { @@ -334,7 +334,7 @@ public actor OverlayFileSystem: FileSystem { try await ensureUpperDirectories(for: destinationPath) try await upper.move(from: sourcePath, to: destinationPath) if await lowerVisibleEntry(sourcePath) != nil { - cuts = cuts.filter { !$0.string.hasPrefix(sourcePath.string + "/") } + cuts = cuts.filter { !$0.isWithin(sourcePath) } cuts.insert(sourcePath) } } @@ -415,7 +415,7 @@ public actor OverlayFileSystem: FileSystem { if cuts.contains(path) { return true } - return cuts.contains { path.string.hasPrefix($0.string + "/") } + return cuts.contains { path.isWithin($0) } } private func upperEntry(_ path: WorkspacePath) async -> FileInfo? { @@ -475,10 +475,10 @@ public actor OverlayFileSystem: FileSystem { } else { throw posixError(ENOENT) } - if target.hasPrefix("/") { + if target.utf8.first == 47 { current = .root } - pending = target.split(separator: "/").map(String.init) + pending + pending = WorkspacePath.splitComponents(target) + pending continue } if !final && info.kind != .directory { diff --git a/Sources/Workspace/RevisionIndex.swift b/Sources/Workspace/RevisionIndex.swift index 4c1814f..3585c55 100644 --- a/Sources/Workspace/RevisionIndex.swift +++ b/Sources/Workspace/RevisionIndex.swift @@ -110,7 +110,7 @@ struct RevisionIndex: Sendable, Equatable { private static func entry(at path: WorkspacePath, in entry: Entry) -> Entry? { if entry.path == path { return entry } guard case let .directory(_, _, children) = entry else { return nil } - for child in children where child.path == path || path.string.hasPrefix(child.path.string + "/") { + for child in children where path.isWithin(child.path) { return self.entry(at: path, in: child) } return nil diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index 84428d7..9a9aca1 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -410,7 +410,7 @@ struct SnapshotJournal { extension Snapshot { static func contains(_ root: WorkspacePath, _ path: WorkspacePath) -> Bool { - root.isRoot || root == path || path.string.hasPrefix(root.string + "/") + path.isWithin(root) } static func replacing(_ tree: Entry, with replacement: Entry) -> Entry { diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 816e7b9..45c2bdc 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -239,7 +239,7 @@ public actor RuleBasedPermissionAuthorizer: PermissionAuthorizing { let paths = endpoints.compactMap { $0 } guard !paths.isEmpty else { return false } let matches: (WorkspacePath) -> Bool = { path in - rule.pathPrefix.isRoot || path == rule.pathPrefix || path.string.hasPrefix(rule.pathPrefix.string + "/") + path.isWithin(rule.pathPrefix) } switch rule.effect { case .allow: return paths.allSatisfy(matches) diff --git a/Sources/Workspace/Support/WorkspacePath.swift b/Sources/Workspace/Support/WorkspacePath.swift index 5637dc5..c6c12a0 100644 --- a/Sources/Workspace/Support/WorkspacePath.swift +++ b/Sources/Workspace/Support/WorkspacePath.swift @@ -14,13 +14,13 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl /// Creates a normalized path from a string literal. public init(stringLiteral value: StringLiteralType) { - precondition(!value.contains("\u{0}"), "workspace path contains a null byte") + precondition(!value.utf8.contains(0), "workspace path contains a null byte") storage = Self.normalizedString(path: value, currentDirectory: Self.root.storage) } /// Creates a validated, normalized path from dynamic text. public init(_ path: String, relativeTo currentDirectory: WorkspacePath = .root) throws { - if path.contains("\u{0}") { + if path.utf8.contains(0) { throw WorkspaceError.invalidPath(path) } storage = Self.normalizedString(path: path, currentDirectory: currentDirectory.storage) @@ -40,7 +40,7 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl /// - Throws: ``WorkspaceError/invalidPath(_:)`` when the input contains unsupported characters. init(validating path: some StringProtocol, relativeTo currentDirectory: WorkspacePath = .root) throws { let string = String(path) - if string.contains("\u{0}") { + if string.utf8.contains(0) { throw WorkspaceError.invalidPath(string) } self.init(normalizing: string, relativeTo: currentDirectory) @@ -70,6 +70,27 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl Self.splitComponents(storage) } + /// Whether this path is equal to or below `ancestor`, using canonical Unicode + /// equality for each component, just like path equality. + func isWithin(_ ancestor: WorkspacePath) -> Bool { + if ancestor.isRoot { return true } + + // The fast path stays allocation-free: UTF-8 views and slices avoid + // constructing component arrays or new strings in containment hot loops. + let bytes = storage.utf8 + let ancestorBytes = ancestor.storage.utf8 + if bytes.starts(with: ancestorBytes) { + let next = bytes.dropFirst(ancestorBytes.count).first + if next == nil || next == 47 { return true } + } + if bytes.allSatisfy({ $0 < 128 }), ancestorBytes.allSatisfy({ $0 < 128 }) { + return false + } + + // Different non-ASCII byte spellings may still be canonically equal. + return components.starts(with: ancestor.components) + } + /// The final path component, or `/` for the root path. var basename: String { Self.basename(storage) @@ -115,14 +136,16 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl } let base: [String] - if path.hasPrefix("/") { + if path.utf8.first == 47 { base = [] } else { base = splitComponents(currentDirectory) } var parts = base - for piece in path.split(separator: "/", omittingEmptySubsequences: true) { + // POSIX separators are bytes, not grapheme clusters. An extending scalar + // after '/' must not hide a component boundary from resolution. + for piece in splitComponents(path) { switch piece { case ".": continue @@ -138,19 +161,15 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl return "/" + parts.joined(separator: "/") } - /// Splits an absolute path string into components, excluding the leading `/`. + /// Splits on UTF-8 slash bytes, omitting empty components and preserving `.` and `..`. static func splitComponents(_ absolutePath: some StringProtocol) -> [String] { - String(absolutePath).split(separator: "/", omittingEmptySubsequences: true).map(String.init) + absolutePath.utf8.split(separator: 47, omittingEmptySubsequences: true) + .map { String(decoding: $0, as: UTF8.self) } } /// Returns the last component of `path`, or `/` for the root path. static func basename(_ path: some StringProtocol) -> String { - let string = String(path) - let normalized = string == "/" ? "/" : string.trimmingCharacters(in: CharacterSet(charactersIn: "/")) - if normalized == "/" || normalized.isEmpty { - return "/" - } - return normalized.split(separator: "/").last.map(String.init) ?? "/" + splitComponents(path).last ?? "/" } /// Returns the parent directory of `path`. @@ -172,11 +191,11 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl static func join(_ lhs: some StringProtocol, _ rhs: some StringProtocol) -> String { let left = String(lhs) let right = String(rhs) - if right.hasPrefix("/") { + if right.utf8.first == 47 { return normalizedString(path: right, currentDirectory: "/") } - let separator = left.hasSuffix("/") ? "" : "/" + let separator = left.utf8.last == 47 ? "" : "/" return normalizedString(path: left + separator + right, currentDirectory: "/") } @@ -188,7 +207,7 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl /// Returns `true` when the token contains glob metacharacters. static func containsGlob(_ token: some StringProtocol) -> Bool { let string = String(token) - return string.contains("*") || string.contains("?") || string.contains("[") + return string.utf8.contains(42) || string.utf8.contains(63) || string.utf8.contains(91) } /// Converts a shell-style glob pattern into an anchored regular expression pattern. @@ -196,49 +215,47 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl /// `*` and `?` match within a single path component (they never cross `/`), `**` matches any /// characters including `/`, and character classes support shell-style negation (`[!abc]`). static func globToRegex(_ pattern: some StringProtocol) -> String { - let pattern = String(pattern) - var regex = "^" - var index = pattern.startIndex - - while index < pattern.endIndex { - let char = pattern[index] - if char == "*" { - let nextIndex = pattern.index(after: index) - if nextIndex < pattern.endIndex, pattern[nextIndex] == "*" { - let afterDoubleStar = pattern.index(after: nextIndex) - if afterDoubleStar < pattern.endIndex, pattern[afterDoubleStar] == "/" { - regex += "(?:.*/)?" - index = afterDoubleStar + let pattern = Array(pattern.utf8) + var regex = Array("^".utf8) + var index = 0 + func append(_ text: String) { regex.append(contentsOf: text.utf8) } + + while index < pattern.count { + let byte = pattern[index] + if byte == 42 { // * + if index + 1 < pattern.count, pattern[index + 1] == 42 { + if index + 2 < pattern.count, pattern[index + 2] == 47 { + append("(?:.*/)?") + index += 2 } else { - regex += ".*" - index = nextIndex + append(".*") + index += 1 } } else { - regex += "[^/]*" + append("[^/]*") } - } else if char == "?" { - regex += "[^/]" - } else if char == "[" { - if let closeIndex = pattern[index...].firstIndex(of: "]"), - closeIndex > pattern.index(after: index) - { - let range = pattern.index(after: index).. index + 1 { + var body = Array(pattern[(index + 1).. ^ + regex.append(91) + regex.append(contentsOf: body) + regex.append(93) index = closeIndex } else { - regex += "\\[" + append("\\[") } + } else if byte < 128 { + append(NSRegularExpression.escapedPattern(for: String(decoding: [byte], as: UTF8.self))) } else { - regex += NSRegularExpression.escapedPattern(for: String(char)) + regex.append(byte) } - index = pattern.index(after: index) + index += 1 } - regex += "$" - return regex + append("$") + return String(decoding: regex, as: UTF8.self) } } diff --git a/Sources/Workspace/UnifiedPatch.swift b/Sources/Workspace/UnifiedPatch.swift index 5a4ed95..ba2cf95 100644 --- a/Sources/Workspace/UnifiedPatch.swift +++ b/Sources/Workspace/UnifiedPatch.swift @@ -54,7 +54,7 @@ extension ChangeSet { } private func patchPath(_ path: WorkspacePath, prefix: String) -> String { - let value = path.string == "/" ? "" : String(path.string.drop(while: { $0 == "/" })) + let value = path.components.joined(separator: "/") return value.isEmpty ? prefix : "\(prefix)/\(value)" } } diff --git a/Sources/Workspace/WorkspaceArchive.swift b/Sources/Workspace/WorkspaceArchive.swift index ae702a3..4ef7c16 100644 --- a/Sources/Workspace/WorkspaceArchive.swift +++ b/Sources/Workspace/WorkspaceArchive.swift @@ -139,16 +139,15 @@ extension Workspace { to destinationRoot: WorkspacePath ) -> WorkspacePath { guard path != sourceRoot else { return destinationRoot } - let prefix = sourceRoot.isRoot ? "/" : sourceRoot.string + "/" - guard path.string.hasPrefix(prefix) else { return path } - return destinationRoot.appending(String(path.string.dropFirst(prefix.count))) + guard path.isWithin(sourceRoot) else { return path } + return destinationRoot.appending(path.components.dropFirst(sourceRoot.components.count).joined(separator: "/")) } private static func entry(at path: WorkspacePath, in entry: Snapshot.Entry) -> Snapshot.Entry { if entry.path == path { return entry } guard case let .directory(directory) = entry else { return .missing(.init(path: path)) } for child in directory.children { - if child.path == path || path.string.hasPrefix(child.path.string + "/") { + if path.isWithin(child.path) { return self.entry(at: path, in: child) } } diff --git a/Sources/Workspace/WorkspaceEvent.swift b/Sources/Workspace/WorkspaceEvent.swift index f05af08..5a85b96 100644 --- a/Sources/Workspace/WorkspaceEvent.swift +++ b/Sources/Workspace/WorkspaceEvent.swift @@ -77,7 +77,7 @@ extension Workspace { guard let path = filter.path else { return true } return changes.touchedPaths.contains { candidate in if filter.recursive { - return candidate == path || candidate.string.hasPrefix(path.isRoot ? "/" : path.string + "/") + return candidate.isWithin(path) } return candidate == path } diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index 1e0b1f1..de1cf94 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -226,7 +226,7 @@ extension Workspace { if entry.path == path { return entry } guard case let .directory(directory) = entry else { return nil } for child in directory.children { - if child.path == path || path.string.hasPrefix(child.path.string + "/") { + if path.isWithin(child.path) { return snapshotEntry(at: path, in: child) } } diff --git a/Tests/WorkspaceTests/UnicodePathRegressionTests.swift b/Tests/WorkspaceTests/UnicodePathRegressionTests.swift new file mode 100644 index 0000000..3ee17bd --- /dev/null +++ b/Tests/WorkspaceTests/UnicodePathRegressionTests.swift @@ -0,0 +1,286 @@ +import Foundation +import Testing +@testable import Workspace + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +@Suite("Unicode path boundaries") +struct UnicodePathRegressionTests { + // All escape fixtures stay inside a disposable container: "outside" means outside + // the LocalFileSystem root, never a real user directory. + private func jail() throws -> (container: URL, outside: URL, fs: LocalFileSystem) { + let container = try TestSupport.temporaryDirectory("UnicodeJail") + let outside = container.appendingPathComponent("outside") + try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true) + return (container, outside, try LocalFileSystem(root: container.appendingPathComponent("root"))) + } + + @Test + func localReadCannotCrossSymlinkedAncestor() async throws { + let fixture = try jail() + defer { TestSupport.remove(fixture.container) } + let secret = Data("outside secret".utf8) + try secret.write(to: fixture.outside.appendingPathComponent("\u{301}secret")) + try await fixture.fs.createSymlink(path: "/l", target: fixture.outside.path) + await #expect(throws: WorkspaceError.self) { + _ = try await fixture.fs.readFile(path: "/l/\u{301}secret") + } + } + + @Test + func localWriteCannotPlantOutsideFile() async throws { + let fixture = try jail() + defer { TestSupport.remove(fixture.container) } + try await fixture.fs.createSymlink(path: "/l", target: fixture.outside.path) + await #expect(throws: WorkspaceError.self) { + try await fixture.fs.writeFile(path: "/l/\u{301}evil.plist", data: Data("payload".utf8), append: false) + } + #expect(try FileManager.default.contentsOfDirectory(atPath: fixture.outside.path).isEmpty) + } + + enum Mutation: CaseIterable, Sendable { + case overwrite, remove, hardLink + } + + @Test(arguments: Mutation.allCases) + func localOtherMutationsCannotCrossSymlinkedAncestor(operation: Mutation) async throws { + let fixture = try jail() + defer { TestSupport.remove(fixture.container) } + let secretURL = fixture.outside.appendingPathComponent("\u{301}secret") + let secret = Data("unchanged".utf8) + try secret.write(to: secretURL) + try await fixture.fs.createSymlink(path: "/l", target: fixture.outside.path) + await #expect(throws: WorkspaceError.self) { + switch operation { + case .overwrite: + try await fixture.fs.writeFile(path: "/l/\u{301}secret", data: Data("changed".utf8), append: false) + case .remove: + try await fixture.fs.remove(path: "/l/\u{301}secret", recursive: false) + case .hardLink: + try await fixture.fs.createHardLink(path: "/stolen", target: "/l/\u{301}secret") + } + } + #expect(try Data(contentsOf: secretURL) == secret) + #expect(!(await fixture.fs.exists(path: "/stolen"))) + } + + @Test + func relativeSymlinkTargetCannotHideAncestor() async throws { + let fixture = try jail() + defer { TestSupport.remove(fixture.container) } + try Data("outside".utf8).write(to: fixture.outside.appendingPathComponent("\u{301}secret")) + try await fixture.fs.createSymlink(path: "/l", target: fixture.outside.path) + try await fixture.fs.createSymlink(path: "/alias", target: "l/\u{301}secret") + await #expect(throws: WorkspaceError.self) { + _ = try await fixture.fs.readFile(path: "/alias") + } + } + + @Test + func absoluteSymlinkTargetPreservesLeadingExtender() async throws { + let root = try TestSupport.temporaryDirectory("UnicodeAbsoluteTarget") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + let target = root.appendingPathComponent("\u{301}file") + try Data("inside".utf8).write(to: target) + try await fs.createSymlink(path: "/alias", target: target.path) + #expect(try await fs.readFile(path: "/alias") == Data("inside".utf8)) + #expect(try await fs.resolveRealPath(path: "/alias") == "/\u{301}file") + } + + @Test + func containmentFastPathPreservesCanonicalEqualityAndBoundaries() { + // These canonically equivalent spellings must reach the non-ASCII fallback. + let decomposed = WorkspacePath("/cafe\u{301}/x") + let composed = WorkspacePath("/caf\u{e9}") + #expect(decomposed.isWithin(composed)) + #expect(composed.isWithin(WorkspacePath("/cafe\u{301}"))) + #expect(!(WorkspacePath("/cafe\u{301}-other/x")).isWithin(composed)) + #expect(!(WorkspacePath("/caf\u{e9}/x")).isWithin(WorkspacePath("/cafe"))) + + // Inclusive equality, root, and slash boundaries use allocation-free byte checks. + let path = WorkspacePath("/private/\u{301}x") + #expect(path.isWithin(path)) + #expect(path.isWithin(.root)) + #expect(path.isWithin("/private")) + #expect(!(WorkspacePath("/private-other/x")).isWithin("/private")) + #expect(!(WorkspacePath("/priv")).isWithin("/private")) + } + + @Test + func denyRulesUseComponentBoundariesAndCanonicalEquality() async throws { + let authorizer = RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.readFile], pathPrefix: "/private", effect: .deny(message: nil)), + .init(operations: [.readFile], pathPrefix: "/caf\u{e9}", effect: .deny(message: nil)), + ], defaultEffect: .allow) + for path: WorkspacePath in ["/private/\u{301}x", "/cafe\u{301}/\u{301}x", "/caf\u{e9}"] { + let decision = await authorizer.authorize(.init(operation: .readFile, path: path)) + guard case .deny = decision else { + Issue.record("deny rule did not match \(path)") + continue + } + } + let decision = await authorizer.authorize(.init(operation: .readFile, path: "/private-other/\u{301}x")) + guard case .allow = decision else { Issue.record("component boundary overmatched"); return } + } + + @Test + func mountRoutesUnicodeChildAndTranslatesRealPath() async throws { + let base = InMemoryFileSystem() + let mounted = InMemoryFileSystem() + try await base.createDirectory(path: "/mnt", recursive: true) + let fs = MountedFileSystem(base: base, mounts: [.init(mountPoint: "/mnt", fileSystem: mounted)]) + try await fs.writeFile(path: "/mnt/\u{301}x", data: Data("mount".utf8), append: false) + #expect(await mounted.exists(path: "/\u{301}x")) + #expect(!(await base.exists(path: "/mnt/\u{301}x"))) + #expect(try await fs.resolveRealPath(path: "/mnt/\u{301}x") == "/mnt/\u{301}x") + let nested = MountedFileSystem(mounts: [.init(mountPoint: "/mnt/\u{301}dir/sub", fileSystem: mounted)]) + #expect(try await nested.listDirectory(path: "/mnt").map(\.name) == ["\u{301}dir"]) + } + + @Test + func overlaySymlinkCannotResurrectCutEntry() async throws { + let root = try TestSupport.temporaryDirectory("UnicodeOverlay") + defer { TestSupport.remove(root) } + let lower = try LocalFileSystem(root: root) + try await lower.createDirectory(path: "/d", recursive: true) + try Data("deleted".utf8).write(to: root.appendingPathComponent("d/\u{301}x")) + for cut: WorkspacePath in ["/d/\u{301}x", "/d"] { + let fs = OverlayFileSystem(over: lower) + try await fs.remove(path: cut, recursive: true) + if cut == "/d" { + // Recreating the directory must not reveal its old lower children. + try await fs.createDirectory(path: "/d", recursive: false) + } + try await fs.createSymlink(path: "/l", target: "d/\u{301}x") + await #expect(throws: (any Error).self) { _ = try await fs.readFile(path: "/l") } + #expect(try await lower.readFile(path: "/d/\u{301}x") == Data("deleted".utf8)) + } + } + + @Test + func inMemoryMoveIntoUnicodeDescendantPreservesSource() async throws { + let fs = InMemoryFileSystem() + try await fs.createDirectory(path: "/a", recursive: true) + try await fs.writeFile(path: "/a/keep", data: Data("keep".utf8), append: false) + await #expect(throws: (any Error).self) { try await fs.move(from: "/a", to: "/a/\u{301}b") } + #expect(await fs.exists(path: "/a")) + #expect(try await fs.readFile(path: "/a/keep") == Data("keep".utf8)) + } + + @Test + func limitedCopyCountsUnicodeDescendants() async throws { + let root = try TestSupport.temporaryDirectory("UnicodeQuota") + defer { TestSupport.remove(root) } + let base = try LocalFileSystem(root: root) + try await base.createDirectory(path: "/a", recursive: true) + // Host fixture makes the physical tree independent of WorkspacePath parsing. + try Data([1, 2]).write(to: root.appendingPathComponent("a/\u{301}x")) + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 3)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 4, limit: 3)) { + try await fs.copy(from: "/a", to: "/b", recursive: true) + } + #expect(!(await base.exists(path: "/b"))) + } + + @Test + func toolJSONCannotBypassDenyRules() async throws { + let base = InMemoryFileSystem() + try await base.createDirectory(path: "/private", recursive: true) + try await base.writeFile(path: "/private/\u{301}x", data: Data("secret".utf8), append: false) + let fs = AuthorizedFileSystem(base: base, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.readFile, .writeFile], pathPrefix: "/private", effect: .deny(message: nil)), + ], defaultEffect: .allow)) + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: fs)) + await #expect(throws: (any Error).self) { + _ = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/private/\u0301x"}"#.utf8)) + } + await #expect(throws: (any Error).self) { + _ = try await adapter.call(name: "workspace_apply", arguments: Data(#"{"edits":[{"kind":"write_text","path":"/private/\u0301x","content":"changed"}]}"#.utf8)) + } + #expect(try await base.readFile(path: "/private/\u{301}x") == Data("secret".utf8)) + } + + @Test(arguments: [false, true]) + func checkpointAndArchiveRetainUnicodeDescendants(diskBacked: Bool) async throws { + let container = try TestSupport.temporaryDirectory("UnicodeRevision") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/d", recursive: true) + try Data("saved".utf8).write(to: root.appendingPathComponent("d/\u{301}x")) + let workspace = Workspace(fileSystem: fs, persistence: diskBacked + ? .directory(container.appendingPathComponent("history")) : .memory) + let checkpoint = try await workspace.createCheckpoint() + #expect(try await workspace.readText("/d/\u{301}x", at: .checkpoint(checkpoint.id)) == "saved") + let archive = try await workspace.archive(at: "/d", from: .checkpoint(checkpoint.id)) + let restored = Workspace() + try await restored.restore(archive, at: "/restored") + #expect(try await restored.readText("/restored/\u{301}x") == "saved") + #expect(!(await restored.exists("/d/\u{301}x"))) + } + + @Test + func recursiveEventsIncludeUnicodeDescendants() async throws { + let workspace = Workspace() + try await workspace.createDirectory("/d") + let stream = await workspace.watchChanges(at: "/d") + let expected = try await workspace.writeText("/d/\u{301}x", "first") + // A second matching event makes an omitted first event observable without + // timing assumptions or an indefinitely suspended iterator on the baseline. + try await workspace.writeText("/d/sentinel", "second") + var iterator = stream.makeAsyncIterator() + #expect(await iterator.next() == .changes(expected)) + } + + @Test + func extendingScalarsAgreeAcrossPathsAndBackends() async throws { + let root = try TestSupport.temporaryDirectory("UnicodeProperties") + defer { TestSupport.remove(root) } + var backends: [(String, any FileSystem)] = [ + ("memory", InMemoryFileSystem()), + ("local", try LocalFileSystem(root: root.appendingPathComponent("local"))), + ("overlay", OverlayFileSystem(over: InMemoryFileSystem())), + ("mounted", MountedFileSystem(mounts: [.init(mountPoint: "/d", fileSystem: InMemoryFileSystem())])), + ("limited", LimitedFileSystem(base: InMemoryFileSystem(), limits: .init(maxTotalBytes: 1000))), + ("authorized", AuthorizedFileSystem(base: InMemoryFileSystem(), authorizer: RuleBasedPermissionAuthorizer(rules: [], defaultEffect: .allow))), + ("sandbox", try SandboxFileSystem(root: .url(root.appendingPathComponent("sandbox")))), + ] + #if canImport(Darwin) + backends.append(("securityScoped", try SecurityScopedFileSystem(url: root.appendingPathComponent("scoped")))) + #endif + let extensions = ["\u{301}", "\u{200d}", "\u{20e3}", "\u{fe0f}", "\u{fe0e}\u{301}", "\u{1f3fb}", "\u{903}"] + for (index, ext) in extensions.enumerated() { + #expect(("/" + ext).count == 1, "fixture must hide slash inside a grapheme cluster") + let name = ext + "x" + let path = try WorkspacePath("/d/" + name) + #expect(path.components == ["d", name]) + #expect(path.name == name) + #expect(path.parent == "/d") + #expect(try WorkspacePath("/" + name, relativeTo: "/other").components == [name]) + #expect(try WorkspacePath("/d/" + name + "/../kept") == "/d/kept") + #expect(WorkspacePath.root.appending("d").appending(name) == path) + for (label, fs) in backends { + do { + try await fs.createDirectory(path: "/d", recursive: true) + let data = Data([UInt8(index)]) + try await fs.writeFile(path: path, data: data, append: false) + #expect(try await fs.listDirectory(path: "/d").contains { $0.name == name }, "\(label)") + #expect(try await fs.readFile(path: path) == data, "\(label)") + let link = try WorkspacePath("/d/link\(index)") + try await fs.createSymlink(path: link, target: name) + #expect(try await fs.readFile(path: link) == data, "\(label)") + #expect(try await fs.resolveRealPath(path: link) == path, "\(label)") + // **/ must be recognized even when / joins the following scalar. + #expect(try await fs.glob(pattern: "/d/**/" + name, currentDirectory: .root).contains(path), "\(label)") + } catch { + Issue.record(error, "\(label), scalar sequence \(index)") + } + } + } + } +} From fdf6d39497d8b7ef269ad74c33d4993a000725d5 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 01:37:33 -0700 Subject: [PATCH 12/34] Make path identity byte-exact and verify stored spellings Workspace used Swift canonical-equivalence equality for path identity while host filesystems applied different case and Unicode-normalization rules. Alternate case spellings could bypass deny rules on case-insensitive volumes such as APFS, while distinct canonical twins on Linux could inherit another path's allow rules and collapse quota accounting. Canonically equivalent sibling names could also trap parented checkpoint saves and revision diffs when String-keyed child maps rejected them as duplicates. This fix makes path identity byte-exact throughout filesystem and history operations. LocalFileSystem verifies spelling per lookup using stored-name attributes or an exact directory-listing match, with known byte-exact filesystems skipping verification. Creations that a filesystem rewrites (HFS+, and listing-verified volumes such as exFAT) are removed and rejected with EILSEQ; rewritten moves are rolled back. Lookup aliases remain ENOENT and conflicting creation or mutation destinations remain EEXIST, without disclosing the stored spelling. --- Package.swift | 4 +- README.md | 65 ++++ .../Workspace/ChangeSet+RevisionIndex.swift | 10 +- Sources/Workspace/ChangeSet+Snapshot.swift | 10 +- Sources/Workspace/CheckpointStore.swift | 18 +- Sources/Workspace/FS/InMemoryFileSystem.swift | 56 +-- Sources/Workspace/FS/LocalFileSystem.swift | 322 +++++++++++++--- Sources/Workspace/FS/LocalPathSpelling.swift | 212 ++++++++++ Sources/Workspace/FS/MountedFileSystem.swift | 14 +- Sources/Workspace/FS/OverlayFileSystem.swift | 8 +- Sources/Workspace/RevisionIndex.swift | 13 + Sources/Workspace/Snapshot.swift | 13 +- .../Workspace/Support/FileChangeStreams.swift | 10 +- Sources/Workspace/Support/PathName.swift | 20 + Sources/Workspace/Support/WorkspacePath.swift | 34 +- Sources/Workspace/Workspace+Merge.swift | 7 +- Sources/Workspace/WorkspaceArchive.swift | 17 +- Sources/Workspace/WorkspaceRevision.swift | 2 +- Sources/WorkspacePOSIX/WorkspacePOSIX.c | 32 ++ .../WorkspacePOSIX/include/WorkspacePOSIX.h | 7 + .../LocalPathSpellingTests.swift | 77 ++++ .../PathIdentityRegressionTests.swift | 363 ++++++++++++++++++ .../WorkspaceTests/PathResolutionTests.swift | 50 ++- .../UnicodePathRegressionTests.swift | 12 +- 24 files changed, 1220 insertions(+), 156 deletions(-) create mode 100644 Sources/Workspace/FS/LocalPathSpelling.swift create mode 100644 Sources/Workspace/Support/PathName.swift create mode 100644 Sources/WorkspacePOSIX/WorkspacePOSIX.c create mode 100644 Sources/WorkspacePOSIX/include/WorkspacePOSIX.h create mode 100644 Tests/WorkspaceTests/LocalPathSpellingTests.swift create mode 100644 Tests/WorkspaceTests/PathIdentityRegressionTests.swift diff --git a/Package.swift b/Package.swift index cc15761..9dc40ea 100644 --- a/Package.swift +++ b/Package.swift @@ -18,8 +18,10 @@ let package = Package( ], targets: [ .target( - name: "Workspace" + name: "Workspace", + dependencies: [.target(name: "WorkspacePOSIX", condition: .when(platforms: [.linux]))] ), + .target(name: "WorkspacePOSIX"), .testTarget( name: "WorkspaceTests", dependencies: ["Workspace"] diff --git a/README.md b/README.md index e978663..39b5457 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,71 @@ invalidations into structured `ChangeSet` values. External changes remain unreco history, matching the direct-filesystem contract above. Directory-backed checkpoint stores use the same push model for cross-instance checkpoint events; there is no periodic polling loop. +## Path Identity + +A `WorkspacePath` is a normalized absolute path whose identity is the exact UTF-8 +bytes of each component. Equality, hashing, ordering, containment, permission +rules, and quota accounting do not fold case or Unicode normalization. For example, +`/café` (NFC) and `/cafe\u{301}` (NFD) are different paths. This deliberately changes +previous releases' use of Swift canonical-equivalence equality. Names and symlink +targets must be valid UTF-8; undecodable host entries are rejected, not repaired +into a different name. In-memory storage +and byte-exact host filesystems can hold both names. + +`LocalFileSystem` requires the requested spelling to match the stored directory +entry. On APFS, including case-sensitive APFS, case or normalization aliases are +rejected: lookups and reads report `ENOENT`; creates, writes, and rename/copy +destinations report `EEXIST` when an equivalent entry exists. Renaming an entry +to an equivalent spelling is rejected too; move through a distinct temporary +name if a spelling change is needed. Case-only renames remain possible on +case-sensitive filesystems. Removing a missing/aliased spelling remains +a no-op. Symlink targets must use the stored spelling, and `resolveRealPath` +returns that spelling. Directory listings (and future bulk traversal) use names +emitted by `readdir`, so they already carry the stored identity. + +On Darwin, volume policy is cached by device. APFS and HFS+ use +`getattrlistat(ATTR_CMN_NAME, FSOPT_NOFOLLOW)` to verify stored names. For APFS, +an `F_GETPATH` check on the held parent **directory** can verify its components +together, against the root descriptor's own path plus the resolved physical +components; the final name still uses `getattrlistat` so hard links remain valid. +Unavailable, overlong, or mismatched directory paths fall back to per-component +checks, so relocating the pinned root or an ancestor remains supported. +No final file descriptor is checked with `F_GETPATH`. + +HFS+ and macOS's exFAT driver store Unicode names in decomposed form (NFD). +Creating a precomposed (NFC) name that decomposes there fails with `EILSEQ` and +the new entry is removed; use the NFD spelling reported by directory listings. +New entries are checked before use on rewriting volumes. A rewritten rename is +rolled back to preserve its source. Other Darwin +filesystems (including exFAT/FSKit, network and unknown types) verify spelling by +an exact byte match in the parent directory's `readdir` listing instead of trusting +name-cache-sensitive attribute APIs. Their new entries are also checked for name +rewriting. Listings are never cached, including on filesystems with coarse mtimes. + +On Linux, ext4/f2fs/bcachefs/overlayfs/tmpfs directory flags are inspected with +`FS_IOC_GETFLAGS`. Directories with `FS_CASEFOLD_FL` use listing verification; +others skip spelling verification. Flags are cached per `(device, inode)` in a +bounded 4,096-entry cache under the existing exclusion of external host namespace +and flag changes. A full cache is cleared before adding a new entry. Workspace +creation/removal invalidates directory entries to prevent stale flags on inode +reuse. Changing casefold requires ownership and an empty directory and is not +exposed by the workspace API. Inspecting flags requires read permission; +unavailable inspection falls back to listing verification, which also requires +read access. While its policy remains cached, a byte-exact directory can still +be traversed with search-only access. + +XFS volume geometry (`XFS_IOC_FSGEOMETRY`) is queried once per device: ascii-ci +volumes use listing verification and ordinary XFS skips it. Known byte-exact btrfs +and ramfs also skip it. Unknown filesystems, and unavailable sensitivity queries, +use listing verification rather than being refused. An existing kernel lookup +without an exact listing match is `ENOENT` for reads or `EEXIST` for creation; +missing creations use exclusive creation. Errors do not disclose the stored name. + +These checks do not make metadata checks and later mutations atomic against an +external process renaming entries; the existing concurrent-host mutation +limitations still apply. Directory names emitted by `readdir` are already verified +spelling and are not looked up a second time solely to check their spelling. + ## Filesystems and Safety Layers Built-in filesystems use consistent names and constructor-driven configuration: diff --git a/Sources/Workspace/ChangeSet+RevisionIndex.swift b/Sources/Workspace/ChangeSet+RevisionIndex.swift index 6758166..8c66f17 100644 --- a/Sources/Workspace/ChangeSet+RevisionIndex.swift +++ b/Sources/Workspace/ChangeSet+RevisionIndex.swift @@ -62,7 +62,7 @@ extension ChangeSet { return ([.init(path: newPath, kind: .file, effect: .modified, diff: result.diff)], result.omission.map { [$0] } ?? []) case let (.symbolicLink(_, oldTarget, oldPermissions), .symbolicLink(newPath, newTarget, newPermissions)): - guard oldTarget != newTarget || oldPermissions != newPermissions else { return ([], []) } + guard !oldTarget.utf8.elementsEqual(newTarget.utf8) || oldPermissions != newPermissions else { return ([], []) } return ([.init(path: newPath, kind: .symlink, effect: .modified)], []) case let (.directory(oldPath, oldPermissions, oldChildren), .directory(newPath, newPermissions, newChildren)): @@ -71,12 +71,12 @@ extension ChangeSet { if oldPermissions != newPermissions { changes.append(.init(path: newPath, kind: .directory, effect: .modified)) } - let oldByName = Dictionary(uniqueKeysWithValues: oldChildren.map { ($0.path.name, $0) }) - let newByName = Dictionary(uniqueKeysWithValues: newChildren.map { ($0.path.name, $0) }) + let oldByName = Dictionary(uniqueKeysWithValues: oldChildren.map { (PathName($0.path.name), $0) }) + let newByName = Dictionary(uniqueKeysWithValues: newChildren.map { (PathName($0.path.name), $0) }) for name in Set(oldByName.keys).union(newByName.keys).sorted() { let result = try await compareIndexEntries( - oldByName[name] ?? .missing(path: oldPath.appending(name)), - newByName[name] ?? .missing(path: newPath.appending(name)), + oldByName[name] ?? .missing(path: oldPath.appending(name.string)), + newByName[name] ?? .missing(path: newPath.appending(name.string)), maxTextBytes: maxTextBytes, loadBefore: loadBefore, loadAfter: loadAfter diff --git a/Sources/Workspace/ChangeSet+Snapshot.swift b/Sources/Workspace/ChangeSet+Snapshot.swift index 1a9e44b..080dd81 100644 --- a/Sources/Workspace/ChangeSet+Snapshot.swift +++ b/Sources/Workspace/ChangeSet+Snapshot.swift @@ -34,18 +34,18 @@ extension ChangeSet { changes.append(.init(path: rhs.path, kind: .file, effect: .modified, diff: diffResult.diff)) if let omission = diffResult.omission { omissions.append(omission) } case let (.symlink(lhs), .symlink(rhs)): - guard lhs.target != rhs.target || lhs.permissions != rhs.permissions else { return } + guard !lhs.target.utf8.elementsEqual(rhs.target.utf8) || lhs.permissions != rhs.permissions else { return } changes.append(.init(path: rhs.path, kind: .symlink, effect: .modified)) case let (.directory(lhs), .directory(rhs)): if lhs.permissions != rhs.permissions { changes.append(.init(path: rhs.path, kind: .directory, effect: .modified)) } - let oldChildren = Dictionary(uniqueKeysWithValues: lhs.children.map { ($0.path.name, $0) }) - let newChildren = Dictionary(uniqueKeysWithValues: rhs.children.map { ($0.path.name, $0) }) + let oldChildren = Dictionary(uniqueKeysWithValues: lhs.children.map { (PathName($0.path.name), $0) }) + let newChildren = Dictionary(uniqueKeysWithValues: rhs.children.map { (PathName($0.path.name), $0) }) for name in Set(oldChildren.keys).union(newChildren.keys).sorted() { compareEntries( - oldChildren[name] ?? .missing(.init(path: lhs.path.appending(name))), - newChildren[name] ?? .missing(.init(path: rhs.path.appending(name))), + oldChildren[name] ?? .missing(.init(path: lhs.path.appending(name.string))), + newChildren[name] ?? .missing(.init(path: rhs.path.appending(name.string))), maxTextBytes: maxTextBytes, changes: &changes, omissions: &omissions diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 6f21b51..712be67 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -298,8 +298,8 @@ actor FileCheckpointStore: CheckpointStore { private let checkpointCapture: Workspace.CheckpointCapturePolicy // Actor reentrancy must suspend contenders before they can conflict with this store's own fd. - private var heldLocks: Set = [] - private var lockWaiters: [String: [CheckedContinuation]] = [:] + private var heldLocks: Set = [] + private var lockWaiters: [PathName: [CheckedContinuation]] = [:] private var listCheckpointsCache: [UUID: (cacheKey: String, checkpoints: [Checkpoint])] = [:] @@ -559,21 +559,21 @@ actor FileCheckpointStore: CheckpointStore { modificationDate: info.modificationDate ) case .directory: - var previousChildren: [String: SnapshotManifest.Node] = [:] + var previousChildren: [PathName: SnapshotManifest.Node] = [:] for child in previous?.kind == .directory ? previous?.children ?? [] : [] { - guard previousChildren.updateValue(child, forKey: child.path.name) == nil else { + guard previousChildren.updateValue(child, forKey: PathName(child.path.name)) == nil else { throw WorkspaceError.storageCorrupted( "snapshot directory \(path) contains duplicate child \(child.path.name)" ) } } var children: [SnapshotManifest.Node] = [] - for entry in try await filesystem.listDirectory(path: path).sorted(by: { $0.name < $1.name }) { + for entry in try await filesystem.listDirectory(path: path).sorted(by: { PathName($0.name) < PathName($1.name) }) { children.append( try await captureManifestNode( from: filesystem, at: path.appending(entry.name), - reusing: previousChildren[entry.name], + reusing: previousChildren[PathName(entry.name)], previousCaptureStartedAt: previousCaptureStartedAt, workspaceId: workspaceId ) @@ -1067,7 +1067,7 @@ actor FileCheckpointStore: CheckpointStore { throw WorkspaceError.storageCorrupted("unexpected artifact: \(url.lastPathComponent)") } return url - }.sorted { $0.lastPathComponent < $1.lastPathComponent } + }.sorted { PathName($0.lastPathComponent) < PathName($1.lastPathComponent) } } private func blobURLs(workspaceId: UUID) throws -> [URL] { @@ -1083,7 +1083,7 @@ actor FileCheckpointStore: CheckpointStore { throw WorkspaceError.storageCorrupted("unexpected entry in blob store: \(url.lastPathComponent)") } return url - }.sorted { $0.lastPathComponent < $1.lastPathComponent } + }.sorted { PathName($0.lastPathComponent) < PathName($1.lastPathComponent) } } private func fileSize(at url: URL) throws -> UInt64 { @@ -1506,11 +1506,13 @@ actor FileCheckpointStore: CheckpointStore { } private func acquireGate(for path: String) async { + let path = PathName(path) if heldLocks.insert(path).inserted { return } await withCheckedContinuation { lockWaiters[path, default: []].append($0) } } private func releaseGate(for path: String) { + let path = PathName(path) guard var waiters = lockWaiters[path], !waiters.isEmpty else { heldLocks.remove(path) return diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index c81da9d..1a7b019 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -17,7 +17,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { private final class Node { enum Kind { case file(Data) - case directory([String: Node]) + case directory([PathName: Node]) case symlink(String) } @@ -98,8 +98,8 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { guard let child = children[name] else { return nil } - let childPath = path.appending(name) - return DirectoryEntry(name: name, info: fileInfo(for: child, path: childPath)) + let childPath = path.appending(name.string) + return DirectoryEntry(name: name.string, info: fileInfo(for: child, path: childPath)) } } @@ -139,20 +139,20 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let (parent, name) = try parentDirectoryAndName(for: writePath) var children = try directoryChildren(of: parent) - if children[name]?.isDirectory == true { + if children[PathName(name)]?.isDirectory == true { throw posixError(EISDIR) } if append, - let existing = children[name], + let existing = children[PathName(name)], case let .file(existingData) = existing.kind { existing.kind = .file(existingData + data) existing.modificationDate = Date() - children[name] = existing + children[PathName(name)] = existing } else { - if let existing = children[name] { unlink(existing) } + if let existing = children[PathName(name)] { unlink(existing) } let node = makeNode(kind: .file(data), permissions: 0o644) - children[name] = node + children[PathName(name)] = node } parent.kind = .directory(children) @@ -172,7 +172,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { var children = try directoryChildren(of: current) let isLast = index == components.count - 1 - if let existing = children[component] { + if let existing = children[PathName(component)] { guard existing.isDirectory else { throw posixError(ENOTDIR) } @@ -188,7 +188,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { } let directory = makeNode(kind: .directory([:]), permissions: 0o755) - children[component] = directory + children[PathName(component)] = directory current.kind = .directory(children) current.modificationDate = Date() current = directory @@ -211,7 +211,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { } var parentChildren = try directoryChildren(of: parent) - parentChildren.removeValue(forKey: name) + parentChildren.removeValue(forKey: PathName(name)) unlink(entry) parent.kind = .directory(parentChildren) parent.modificationDate = Date() @@ -235,28 +235,28 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let (destinationParent, destinationName) = try parentDirectoryAndName(for: destinationPath) if sourceParent === destinationParent { var children = try directoryChildren(of: sourceParent) - if children[destinationName] != nil { + if children[PathName(destinationName)] != nil { throw posixError(EEXIST) } - children.removeValue(forKey: sourceName) - children[destinationName] = sourceNode + children.removeValue(forKey: PathName(sourceName)) + children[PathName(destinationName)] = sourceNode sourceParent.kind = .directory(children) sourceParent.modificationDate = Date() return } var destinationChildren = try directoryChildren(of: destinationParent) - if destinationChildren[destinationName] != nil { + if destinationChildren[PathName(destinationName)] != nil { throw posixError(EEXIST) } var sourceChildren = try directoryChildren(of: sourceParent) - sourceChildren.removeValue(forKey: sourceName) + sourceChildren.removeValue(forKey: PathName(sourceName)) sourceParent.kind = .directory(sourceChildren) sourceParent.modificationDate = Date() - destinationChildren[destinationName] = sourceNode + destinationChildren[PathName(destinationName)] = sourceNode destinationParent.kind = .directory(destinationChildren) destinationParent.modificationDate = Date() } @@ -272,11 +272,11 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let (destinationParent, destinationName) = try parentDirectoryAndName(for: destinationPath) var destinationChildren = try directoryChildren(of: destinationParent) - if destinationChildren[destinationName] != nil { + if destinationChildren[PathName(destinationName)] != nil { throw posixError(EEXIST) } - destinationChildren[destinationName] = clone(sourceNode) + destinationChildren[PathName(destinationName)] = clone(sourceNode) destinationParent.kind = .directory(destinationChildren) destinationParent.modificationDate = Date() } @@ -290,11 +290,11 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let (parent, name) = try parentDirectoryAndName(for: path) var children = try directoryChildren(of: parent) - if children[name] != nil { + if children[PathName(name)] != nil { throw posixError(EEXIST) } - children[name] = makeNode(kind: .symlink(target), permissions: 0o777) + children[PathName(name)] = makeNode(kind: .symlink(target), permissions: 0o777) parent.kind = .directory(children) parent.modificationDate = Date() } @@ -312,12 +312,12 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let (parent, name) = try parentDirectoryAndName(for: path) var children = try directoryChildren(of: parent) - if children[name] != nil { + if children[PathName(name)] != nil { throw posixError(EEXIST) } sourceNode.linkCount += 1 - children[name] = sourceNode + children[PathName(name)] = sourceNode parent.kind = .directory(children) parent.modificationDate = Date() } @@ -416,7 +416,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { } for (name, child) in children.sorted(by: { $0.key < $1.key }) { - let childPath = currentPath.appending(name) + let childPath = currentPath.appending(name.string) paths.append(childPath) collectPaths(node: child, currentPath: childPath, into: &paths) } @@ -444,7 +444,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { ) } - private func directoryChildren(of node: Node) throws -> [String: Node] { + private func directoryChildren(of node: Node) throws -> [PathName: Node] { guard case let .directory(children) = node.kind else { throw posixError(ENOTDIR) } @@ -466,7 +466,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { private func parentDirectoryEntryIfPresent(for path: WorkspacePath) throws -> (Node, String, Node)? { let (parent, name) = try parentDirectoryAndName(for: path) let children = try directoryChildren(of: parent) - guard let entry = children[name] else { + guard let entry = children[PathName(name)] else { return nil } return (parent, name, entry) @@ -490,7 +490,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { throw posixError(ENOTDIR) } - guard let child = children[component] else { + guard let child = children[PathName(component)] else { throw posixError(ENOENT) } @@ -535,7 +535,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { throw posixError(ENOTDIR) } - guard let child = children[component] else { + guard let child = children[PathName(component)] else { if allowMissing { return resolvedPath.appending(components.dropFirst(index).joined(separator: "/")) } diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index d343203..ad6e008 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -24,14 +24,19 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem typealias ExclusiveRename = @Sendable (Int32, String, Int32, String) -> Int32 private let fileManager: FileManager + private let spelling = LocalPathSpelling() private let exclusiveRename: ExclusiveRename private let stateLock = NSLock() private var configuration: ConfigurationSnapshot? private final class Descriptor: @unchecked Sendable { let value: Int32 - init(_ value: Int32) { + let spellingPolicy: LocalPathSpelling.Policy + let directoryInfo: LocalStat? + init(_ value: Int32, spellingPolicy: LocalPathSpelling.Policy, directoryInfo: LocalStat? = nil) { self.value = value + self.spellingPolicy = spellingPolicy + self.directoryInfo = directoryInfo } deinit { close(value) @@ -43,6 +48,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let resolvedRootPath: String let foundationRootPath: String let root: Descriptor + let rootSpelling: String? } // Glibc does not expose O_PATH without _GNU_SOURCE. @@ -58,6 +64,8 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let parent: Descriptor let name: String let path: WorkspacePath + var isVerified = false + var directoryInfo: LocalStat? // Every variable *at() name comes from an immutable, validated Entry. // Even a future parser bug must not pass a multi-component path to the kernel. @@ -68,11 +76,24 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem self.path = path } + // Names emitted by readdir already have stored spelling. This initializer + // still applies the jail's byte-level single-component validation. + init(parent: Descriptor, storedName: String, path: WorkspacePath) throws { + try self.init(parent: parent, name: storedName, path: path) + isVerified = true + } + + func requireVerified() throws { + guard isVerified else { throw WorkspaceError.invalidPath(path.string) } + } + // The only permitted dot entry refers to an already resolved directory fd. init(directory: Descriptor, path: WorkspacePath) { self.parent = directory self.name = "." self.path = path + self.isVerified = true + self.directoryInfo = directory.directoryInfo } } @@ -122,7 +143,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let rootFD = open(canonical, Self.searchDirectoryFlags | O_NOFOLLOW | O_CLOEXEC) let openError = errno try check(rootFD, errorCode: openError) - let root = Descriptor(rootFD) + let root = try directoryDescriptor(rootFD) stateLock.lock() defer { stateLock.unlock() @@ -131,7 +152,8 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem rootURL: url, resolvedRootPath: canonical, foundationRootPath: url.resolvingSymlinksInPath().standardizedFileURL.path, - root: root + root: root, + rootSpelling: root.spellingPolicy == .darwin ? Self.directoryPath(root.value) : nil ) } @@ -152,13 +174,42 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } } + private func directoryDescriptor(_ fd: Int32, info suppliedInfo: LocalStat? = nil) throws -> Descriptor { + do { + var info = suppliedInfo ?? LocalStat() + #if canImport(Glibc) + // Flags belong to this opened inode. A concurrent workspace rename + // can replace a previously statted directory with a casefolded one. + try check(fstat(fd, &info)) + #else + if suppliedInfo == nil { try check(fstat(fd, &info)) } + #endif + return Descriptor(fd, spellingPolicy: try spelling.policy(for: fd, info: info), directoryInfo: info) + } + catch { close(fd); throw error } + } + + private func checkSpelling(_ entry: Entry, creating: Bool = false) throws { + try spelling.check(parent: entry.parent.value, name: entry.name, + policy: entry.parent.spellingPolicy, creating: creating) + } + + private func verifiedEntry(_ entry: Entry, creating: Bool = false) throws -> Entry { + try checkSpelling(entry, creating: creating) + var verified = entry + verified.isVerified = true + return verified + } + private func metadata(_ entry: Entry) throws -> LocalStat { + try entry.requireVerified() var value = LocalStat() try check(fstatat(entry.parent.value, entry.name, &value, AT_SYMLINK_NOFOLLOW)) return value } private func linkTarget(_ entry: Entry) throws -> String { + try entry.requireVerified() var bytes = [UInt8](repeating: 0, count: 256) while true { let count = bytes.withUnsafeMutableBytes { @@ -169,7 +220,10 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } if count < bytes.count { - return String(decoding: bytes.prefix(count), as: UTF8.self) + guard let target = String(bytes: bytes.prefix(count), encoding: .utf8) else { + throw WorkspaceError.invalidPath("non-UTF-8 symlink target at \(entry.path)") + } + return target } bytes = [UInt8](repeating: 0, count: bytes.count * 2) } @@ -183,41 +237,93 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let targetComponents = WorkspacePath.splitComponents(target) for root in [configuration.resolvedRootPath, configuration.foundationRootPath, configuration.rootURL.path] { let rootComponents = WorkspacePath.splitComponents(root) - if targetComponents.starts(with: rootComponents) { + if targetComponents.starts(with: rootComponents, by: { $0.utf8.elementsEqual($1.utf8) }) { return Array(targetComponents.dropFirst(rootComponents.count)) } } throw WorkspaceError.invalidPath(target) } + private enum ResolutionRetry: Error { case componentChecks } + + /// F_GETPATH is used only for APFS directories, never final file descriptors: + /// directories have one physical path, whereas a file may have many hard links. + private static func directoryPath(_ fd: Int32) -> String? { + #if canImport(Darwin) + return withUnsafeTemporaryAllocation(of: CChar.self, capacity: Int(MAXPATHLEN)) { buffer in + guard fcntl(fd, F_GETPATH, buffer.baseAddress!) == 0 else { return nil } + return String(validatingCString: buffer.baseAddress!) + } + #else + return nil + #endif + } + + private func verifyParent( + _ directory: Descriptor, names: [String], configuration: ConfigurationSnapshot, + batchAPFS: Bool + ) throws { + guard batchAPFS, !names.isEmpty else { return } + guard directory.spellingPolicy == .darwin, + let root = configuration.rootSpelling, + let stored = Self.directoryPath(directory.value) else { + throw ResolutionRetry.componentChecks + } + let expected = (root == "/" ? "" : root) + "/" + names.joined(separator: "/") + #if canImport(Darwin) + guard expected.utf8.count < Int(MAXPATHLEN) else { throw ResolutionRetry.componentChecks } + #endif + guard stored.utf8.elementsEqual(expected.utf8) else { + // The host may have relocated the pinned root or an ancestor since + // configuration. Only component checks can distinguish that from an alias. + throw ResolutionRetry.componentChecks + } + } + private func resolve( _ path: WorkspacePath, followFinal: Bool, createParents: Bool = false, - allowMissingAncestors: Bool = false, + allowMissingAncestors: Bool = false, creating: Bool = false, configuration: ConfigurationSnapshot? = nil ) throws -> Entry { let config = try configuration ?? requireConfiguration() - guard !path.string.utf8.contains(0) else { - throw WorkspaceError.invalidPath(path.string) + do { + return try resolveComponents(path, followFinal: followFinal, createParents: createParents, + allowMissingAncestors: allowMissingAncestors, creating: creating, configuration: config, + batchAPFS: config.rootSpelling != nil) + } catch is ResolutionRetry { + // F_GETPATH failure/mismatch/overflow or crossing a non-APFS mount: do not trust + // an incomplete path. Retry descriptor-relative per-component checks. + return try resolveComponents(path, followFinal: followFinal, createParents: createParents, + allowMissingAncestors: allowMissingAncestors, creating: creating, configuration: config, + batchAPFS: false) } + } + + private func resolveComponents( + _ path: WorkspacePath, followFinal: Bool, createParents: Bool, + allowMissingAncestors: Bool, creating: Bool, configuration config: ConfigurationSnapshot, + batchAPFS: Bool + ) throws -> Entry { + guard !path.string.utf8.contains(0) else { throw WorkspaceError.invalidPath(path.string) } var pending = path.components - var current = config.root + let root = config.root + var current = root var names: [String] = [] var missingAncestor = false var hops = 0 while !pending.isEmpty { let name = pending.removeFirst() - if name == "." || name.isEmpty { - continue - } + if name == "." || name.isEmpty { continue } if name == ".." { - guard !names.isEmpty else { - throw WorkspaceError.invalidPath(path.string) + guard !names.isEmpty else { throw WorkspaceError.invalidPath(path.string) } + if !missingAncestor { + try verifyParent(current, names: names, configuration: config, batchAPFS: batchAPFS) } names.removeLast() - current = config.root + current = root missingAncestor = false for ancestor in names { - let entry = try Entry(parent: current, name: ancestor, path: .root) + let entry = try verifiedEntry(Entry(parent: current, name: ancestor, path: .root), creating: creating) do { current = try openEntry(entry, flags: Self.searchDirectoryFlags) } catch let error as NSError where allowMissingAncestors @@ -229,69 +335,152 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem continue } let final = pending.isEmpty - let entry = try Entry( - parent: current, name: name, + var entry = try Entry(parent: current, name: name, path: WorkspacePath(unchecked: "/" + (names + [name]).joined(separator: "/"))) - if final && !followFinal { - return entry - } - // Only potential-path resolution permits missing ancestors. The descriptor is - // no longer an operation target; this mode consumes only the resulting path. if missingAncestor { + // Only potential-path resolution consumes unverified entries, and + // only their path strings. I/O helpers require a verified Entry. if final { return entry } names.append(name) continue } - var info: LocalStat - do { - info = try metadata(entry) - } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { - if final { - return entry + if !batchAPFS { entry = try verifiedEntry(entry, creating: creating) } + if final && !followFinal { + if batchAPFS { + try verifyParent(current, names: names, configuration: config, batchAPFS: true) + entry = try verifiedEntry(entry, creating: creating) + } + return entry + } + var info = LocalStat() + let status = fstatat(entry.parent.value, entry.name, &info, AT_SYMLINK_NOFOLLOW) + let statusError = errno + // Directory traversal may defer spelling checks until the parent fd is + // verified as a whole. Before reading a link, returning a target, or + // creating anything, both the parent and current name are verified. + let traverseDirectory = status == 0 && info.st_mode & S_IFMT == S_IFDIR && !final + if batchAPFS && !traverseDirectory { + try verifyParent(current, names: names, configuration: config, batchAPFS: true) + entry = try verifiedEntry(entry, creating: creating) + } + if status < 0 { + guard statusError == ENOENT else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(statusError)) } + if final { return entry } if allowMissingAncestors { names.append(name) missingAncestor = true continue } - guard createParents else { - throw error - } + guard createParents else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } let result = mkdirat(entry.parent.value, entry.name, 0o755) - if result < 0 && errno != EEXIST { - try check(result) - } + if result < 0 && errno != EEXIST { try check(result) } + if result == 0 { try validateCreated(entry, directory: true) } + else { try checkSpelling(entry, creating: true) } info = try metadata(entry) } if info.st_mode & S_IFMT == S_IFLNK { hops += 1 - guard hops <= 40 else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) - } + guard hops <= 40 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } let target = try linkTarget(entry) if target.utf8.first == 47 { pending = try absoluteComponents(target, configuration: config) + pending - current = config.root + current = root names = [] } else { pending = WorkspacePath.splitComponents(target) + pending } continue } - if final { - return entry + if info.st_mode & S_IFMT == S_IFDIR { entry.directoryInfo = info } + if final { return entry } + if batchAPFS && !entry.isVerified { + // This is the only open of an unverified Entry. It opens a directory + // with O_NOFOLLOW, retains its mandatory policy, and cannot expose + // data or mutate it until verifyParent succeeds at the boundary. + let fd = openat(current.value, name, Self.searchDirectoryFlags | O_NOFOLLOW | O_CLOEXEC) + if fd < 0 { + let error = errno + try verifyParent(current, names: names, configuration: config, batchAPFS: true) + _ = try verifiedEntry(entry, creating: creating) + throw NSError(domain: NSPOSIXErrorDomain, code: Int(error)) + } + current = try directoryDescriptor(fd, info: info) + if current.spellingPolicy != .darwin { throw ResolutionRetry.componentChecks } + } else { + current = try openEntry(entry, flags: Self.searchDirectoryFlags, directoryInfo: info) } - current = try openEntry(entry, flags: Self.searchDirectoryFlags) names.append(name) } + if !missingAncestor { + try verifyParent(current, names: names, configuration: config, batchAPFS: batchAPFS) + } return Entry(directory: current, path: WorkspacePath(unchecked: "/" + names.joined(separator: "/"))) } - private func openEntry(_ entry: Entry, flags: Int32, mode: mode_t = 0o666) throws -> Descriptor { - let fd = openat(entry.parent.value, entry.name, flags | O_NOFOLLOW | O_CLOEXEC, mode) + private func openEntry( + _ entry: Entry, flags: Int32, mode: mode_t = 0o666, directoryInfo: LocalStat? = nil + ) throws -> Descriptor { + try entry.requireVerified() + var openFlags = flags | O_NOFOLLOW | O_CLOEXEC + let verifyCreation = flags & O_CREAT != 0 && entry.parent.spellingPolicy.verifiesCreation + if verifyCreation { openFlags |= O_EXCL } + var fd = openat(entry.parent.value, entry.name, openFlags, mode) + let created = verifyCreation && fd >= 0 + if fd < 0 && errno == EEXIST && verifyCreation && flags & O_EXCL == 0 { + // Open an existing entry without O_CREAT: cleanup must never unlink + // an old file, even if another process removes it between these calls. + fd = openat(entry.parent.value, entry.name, (openFlags & ~(O_CREAT | O_EXCL)), mode) + } let openError = errno try check(fd, errorCode: openError) - return Descriptor(fd) + if created { + do { try validateCreated(entry, directory: false) } + catch { close(fd); throw error } + } + if flags & O_DIRECTORY != 0 { + return try directoryDescriptor(fd, info: directoryInfo ?? entry.directoryInfo) + } + return Descriptor(fd, spellingPolicy: .exact) // non-directory fd; never a resolution parent + } + + /// HFS+ can rewrite a new name; unknown filesystems must prove preservation too. + /// This is called only after a successful exclusive creation, before filling it. + private func validateCreated(_ entry: Entry, directory: Bool) throws { + if entry.parent.spellingPolicy.verifiesCreation { + do { + try spelling.check(parent: entry.parent.value, name: entry.name, + policy: entry.parent.spellingPolicy, creating: true, afterCreate: true) + } catch { + let result = unlinkat(entry.parent.value, entry.name, directory ? AT_REMOVEDIR : 0) + if result < 0 { + throw WorkspaceRollbackError(originalError: error, + rollbackError: NSError(domain: NSPOSIXErrorDomain, code: Int(errno))) + } + throw error + } + } + if directory { + // Inode reuse after a workspace-created directory must not reuse flags + // cached for an earlier directory, potentially under a different parent. + spelling.forget(try metadata(entry)) + } + } + + private func validateMoved(_ source: Entry, to destination: Entry) throws { + guard destination.parent.spellingPolicy.verifiesCreation else { return } + do { + try spelling.check(parent: destination.parent.value, name: destination.name, + policy: destination.parent.spellingPolicy, creating: true, afterCreate: true) + } catch { + // A rename is not a disposable creation. Preserve the source on rewrite. + if renameat(destination.parent.value, destination.name, source.parent.value, source.name) < 0 { + throw WorkspaceRollbackError(originalError: error, + rollbackError: NSError(domain: NSPOSIXErrorDomain, code: Int(errno))) + } + throw error + } } private func fileInfo(_ info: LocalStat, path: WorkspacePath) -> FileInfo { @@ -341,14 +530,15 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } let name = withUnsafePointer(to: &item.pointee.d_name) { $0.withMemoryRebound(to: CChar.self, capacity: MemoryLayout.size(ofValue: item.pointee.d_name)) { - String(cString: $0) + String(validatingCString: $0) } } + guard let name else { throw WorkspaceError.invalidPath("non-UTF-8 directory entry") } if name != "." && name != ".." { result.append(name) } } - return result.sorted() + return result.sorted { $0.utf8.lexicographicallyPrecedes($1.utf8) } } /// See ``FileSystem/listDirectory(path:)``. @@ -357,7 +547,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem return try names(in: directory).map { name in let child = path.appending(name) return DirectoryEntry( - name: name, info: fileInfo(try metadata(Entry(parent: directory, name: name, path: child)), path: child) + name: name, info: fileInfo(try metadata(Entry(parent: directory, storedName: name, path: child)), path: child) ) } } @@ -455,7 +645,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem /// See ``FileSystem/createFile(path:data:)``. public func createFile(path: WorkspacePath, data: Data) async throws { - let entry = try resolve(path, followFinal: false, createParents: true) + let entry = try resolve(path, followFinal: false, createParents: true, creating: true) try write(data, to: openEntry(entry, flags: O_WRONLY | O_CREAT | O_EXCL)) } @@ -465,7 +655,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem /// breaking its hard-link sharing. Links that escape the root are rejected. /// Appends preserve the opened inode. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { - let entry = try resolve(path, followFinal: true, createParents: true) + let entry = try resolve(path, followFinal: true, createParents: true, creating: true) if append { try write(data, to: openEntry(entry, flags: O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK)) } else { @@ -485,25 +675,29 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } private func replace(_ entry: Entry, body: (Descriptor) throws -> Void) throws { + try entry.requireVerified() let temporary = ".workspace-" + UUID().uuidString - let temporaryEntry = try Entry(parent: entry.parent, name: temporary, path: entry.path) + let temporaryEntry = try verifiedEntry(Entry(parent: entry.parent, name: temporary, path: entry.path), creating: true) let fd = try openEntry(temporaryEntry, flags: O_WRONLY | O_CREAT | O_EXCL) defer { _ = unlinkat(temporaryEntry.parent.value, temporaryEntry.name, 0) } try body(fd) + try checkSpelling(entry, creating: true) try check(renameat(temporaryEntry.parent.value, temporaryEntry.name, entry.parent.value, entry.name)) + try validateCreated(entry, directory: false) } /// See ``FileSystem/createDirectory(path:recursive:)``. public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { - let entry = try resolve(path, followFinal: true, createParents: recursive) + let entry = try resolve(path, followFinal: true, createParents: recursive, creating: true) let result = mkdirat(entry.parent.value, entry.name, 0o755) if result < 0 && errno == EEXIST && recursive { _ = try openEntry(entry, flags: Self.searchDirectoryFlags) return } try check(result) + try validateCreated(entry, directory: true) } private func removeEntry(_ entry: Entry, recursive: Bool) throws { @@ -515,13 +709,14 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } if info.st_mode & S_IFMT == S_IFDIR { if recursive { - let directory = try openEntry(entry, flags: O_RDONLY | O_DIRECTORY) + let directory = try openEntry(entry, flags: O_RDONLY | O_DIRECTORY, directoryInfo: info) for name in try names(in: directory) { try removeEntry( - Entry(parent: directory, name: name, path: entry.path.appending(name)), recursive: true) + Entry(parent: directory, storedName: name, path: entry.path.appending(name)), recursive: true) } } try check(unlinkat(entry.parent.value, entry.name, AT_REMOVEDIR)) + spelling.forget(info) } else { try check(unlinkat(entry.parent.value, entry.name, 0)) } @@ -552,9 +747,10 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { let config = try requireConfiguration() let source = try resolve(sourcePath, followFinal: false, configuration: config) - let destination = try resolve(destinationPath, followFinal: false, createParents: true, configuration: config) + let destination = try resolve(destinationPath, followFinal: false, createParents: true, creating: true, configuration: config) let renameError = exclusiveRename(source.parent.value, source.name, destination.parent.value, destination.name) if renameError == 0 { + try validateMoved(source, to: destination) return } #if canImport(Darwin) @@ -574,6 +770,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(ENOENT) { // Reuse the resolved descriptors. Only no-clobber atomicity is relaxed, not containment. try check(renameat(source.parent.value, source.name, destination.parent.value, destination.name)) + try validateMoved(source, to: destination) return } throw NSError(domain: NSPOSIXErrorDomain, code: Int(EEXIST)) @@ -607,18 +804,20 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } private func copyEntry(_ source: Entry, to destination: Entry, recursive: Bool) throws { + let destination = try verifiedEntry(destination, creating: true) let info = try metadata(source) switch info.st_mode & S_IFMT { case S_IFDIR: guard recursive else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) } - let input = try openEntry(source, flags: O_RDONLY | O_DIRECTORY) + let input = try openEntry(source, flags: O_RDONLY | O_DIRECTORY, directoryInfo: info) try check(mkdirat(destination.parent.value, destination.name, 0o755)) + try validateCreated(destination, directory: true) let output = try openEntry(destination, flags: O_RDONLY | O_DIRECTORY) for name in try names(in: input) { try copyEntry( - Entry(parent: input, name: name, path: source.path.appending(name)), + Entry(parent: input, storedName: name, path: source.path.appending(name)), to: Entry(parent: output, name: name, path: destination.path.appending(name)), recursive: true) } try check(fchmod(output.value, info.st_mode & 0o7777)) @@ -626,6 +825,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem let target = try linkTarget(source) try removeEntry(destination, recursive: false) try check(symlinkat(target, destination.parent.value, destination.name)) + try validateCreated(destination, directory: false) case S_IFREG: let input = try openEntry(source, flags: O_RDONLY | O_NONBLOCK) let handle = FileHandle(fileDescriptor: try duplicateDescriptor(input.value), closeOnDealloc: true) @@ -648,7 +848,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { let config = try requireConfiguration() let source = try resolve(sourcePath, followFinal: false, configuration: config) - let destination = try resolve(destinationPath, followFinal: false, createParents: true, configuration: config) + let destination = try resolve(destinationPath, followFinal: false, createParents: true, creating: true, configuration: config) guard !destination.path.isWithin(source.path) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EINVAL)) @@ -659,17 +859,19 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem /// See ``FileSystem/createSymlink(path:target:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { _ = try WorkspacePath(validating: target, relativeTo: path.dirname) - let entry = try resolve(path, followFinal: false, createParents: true) + let entry = try resolve(path, followFinal: false, createParents: true, creating: true) try check(symlinkat(target, entry.parent.value, entry.name)) + try validateCreated(entry, directory: false) } /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { let config = try requireConfiguration() let source = try resolve(target, followFinal: true, configuration: config) - let destination = try resolve(path, followFinal: false, createParents: true, configuration: config) + let destination = try resolve(path, followFinal: false, createParents: true, creating: true, configuration: config) // No AT_SYMLINK_FOLLOW: a concurrent final-component swap must not dereference a link. try check(linkat(source.parent.value, source.name, destination.parent.value, destination.name, 0)) + try validateCreated(destination, directory: false) } /// See ``FileSystem/readSymlink(path:)``. diff --git a/Sources/Workspace/FS/LocalPathSpelling.swift b/Sources/Workspace/FS/LocalPathSpelling.swift new file mode 100644 index 0000000..fe8a8ff --- /dev/null +++ b/Sources/Workspace/FS/LocalPathSpelling.swift @@ -0,0 +1,212 @@ +import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +import WorkspacePOSIX +#endif + +/// Volume properties and Linux directory flags are stable under the documented +/// exclusion of concurrent host namespace/flag changes. Listings are never cached. +final class LocalPathSpelling: @unchecked Sendable { + enum Policy { + case exact, darwin, hfs, verifyByListing + + var verifiesCreation: Bool { self == .hfs || self == .verifyByListing } + } + private enum Volume { case policy(Policy), linuxFlags } + private struct DirectoryID: Hashable { + let device: UInt64 + let inode: UInt64 + init(_ info: stat) { + device = UInt64(truncatingIfNeeded: info.st_dev) + inode = UInt64(info.st_ino) + } + } + private let lock = NSLock() + private var volumes: [UInt64: Volume] = [:] + private var directories: [DirectoryID: Policy] = [:] + + func forget(_ info: stat) { + lock.lock() + directories.removeValue(forKey: DirectoryID(info)) + lock.unlock() + } + + func policy(for directory: Int32, info suppliedInfo: stat? = nil) throws -> Policy { + var info = suppliedInfo ?? stat() + if suppliedInfo == nil, fstat(directory, &info) != 0 { throw posix(errno) } + let id = DirectoryID(info) + lock.lock() + let cached = volumes[id.device] + let directoryPolicy = directories[id] + lock.unlock() + if let directoryPolicy { return directoryPolicy } + let volume: Volume + if let cached { volume = cached } else { + volume = classify(directory) + lock.lock() + volumes[id.device] = volume + lock.unlock() + } + switch volume { + case let .policy(policy): return policy + case .linuxFlags: + #if canImport(Glibc) + // O_PATH cannot service FS_IOC_GETFLAGS. Reopen the pinned directory + // once per inode, not once per operation. Unavailable flags use listing. + let fd = openat(directory, ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC) + guard fd >= 0 else { return .verifyByListing } + defer { close(fd) } + var flags: UInt32 = 0 + guard workspace_directory_flags(fd, &flags) == 0 else { return .verifyByListing } + let policy: Policy = flags & 0x4000_0000 == 0 ? .exact : .verifyByListing // FS_CASEFOLD_FL + lock.lock() + // Bound memory on long-lived workspaces traversing many directories. + // Eviction is safe: the next access simply inspects the flags again. + if directories.count >= 4_096, directories[id] == nil { + directories.removeAll(keepingCapacity: true) + } + directories[id] = policy + lock.unlock() + return policy + #else + return .verifyByListing + #endif + } + } + + private func classify(_ fd: Int32) -> Volume { + #if canImport(Darwin) + var info = statfs() + guard fstatfs(fd, &info) == 0 else { return .policy(.verifyByListing) } + let capacity = MemoryLayout.size(ofValue: info.f_fstypename) + let type = withUnsafePointer(to: &info.f_fstypename) { + $0.withMemoryRebound(to: CChar.self, capacity: capacity) { String(cString: $0) } + } + if type == "apfs" { return .policy(.darwin) } + if type == "hfs" { return .policy(.hfs) } + // In particular, exFAT/FSKit getattrlistat may report a primed lookup + // alias, not the stored name. Only APFS and HFS use that API as evidence. + return .policy(.verifyByListing) + #else + var type: UInt64 = 0 + guard workspace_filesystem_type(fd, &type) == 0 else { return .policy(.verifyByListing) } + switch type { + case 0xEF53, 0xF2F5_2010, 0xCA45_1A4E, 0x794C_7630, 0x0102_1994: + return .linuxFlags // ext*, f2fs, bcachefs, overlayfs, tmpfs (casefold since 6.13) + case 0x9123_683E, 0x8584_58F6: + return .policy(.exact) // btrfs, ramfs + case 0x5846_5342: // XFS ascii-ci is a volume property, not a directory flag. + let readable = openat(fd, ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC) + guard readable >= 0 else { return .policy(.verifyByListing) } + defer { close(readable) } + var insensitive: Int32 = 0 + guard workspace_xfs_case_insensitive(readable, &insensitive) == 0 else { + return .policy(.verifyByListing) + } + return .policy(insensitive == 0 ? .exact : .verifyByListing) + default: return .policy(.verifyByListing) + } + #endif + } + + /// Aliases are ENOENT on lookup and EEXIST at mutation destinations. Never + /// include the stored spelling in errors: authorization may have denied it. + func check(parent: Int32, name: String, policy: Policy, creating: Bool, afterCreate: Bool = false) throws { + guard name != "." else { return } // trusted resolved-directory sentinel + switch policy { + case .exact: return + case .verifyByListing: + var info = stat() + let result = fstatat(parent, name, &info, AT_SYMLINK_NOFOLLOW) + if result < 0 && errno != ENOENT { throw posix(errno) } + try Self.checkListing(name: name, exists: result == 0, creating: creating, afterCreate: afterCreate) { + try Self.listNames(parent) + } + case .darwin, .hfs: + #if canImport(Darwin) + var attributes = attrlist() + attributes.bitmapcount = UInt16(ATTR_BIT_MAP_COUNT) + attributes.commonattr = attrgroup_t(ATTR_CMN_NAME) + try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 2048) { buffer in + let result = getattrlistat(parent, name, &attributes, buffer.baseAddress!, buffer.count, UInt(FSOPT_NOFOLLOW)) + if result < 0 { + if errno == ENOENT && !afterCreate { return } + throw posix(errno) + } + let bytes = UnsafeRawBufferPointer(buffer) + let length = Int(bytes.loadUnaligned(as: UInt32.self)) + let ref = bytes.loadUnaligned(fromByteOffset: 4, as: attrreference_t.self) + let start = 4 + Int(ref.attr_dataoffset) + let count = Int(ref.attr_length) + guard length <= buffer.count, start >= 12, count > 0, + start <= length, count <= length - start, buffer[start + count - 1] == 0 else { + throw posix(EIO) + } + guard bytes[start..<(start + count - 1)].elementsEqual(name.utf8) else { + throw Self.spellingMismatch(creating: creating, afterCreate: afterCreate) + } + } + #else + throw posix(ENOTSUP) + #endif + } + } + + /// Shared by the real directory stream and fake listing sources in tests. + /// An existing kernel lookup is not spelling evidence: its dentry may be an alias. + static func checkListing( + name: String, exists: Bool, creating: Bool, afterCreate: Bool = false, + list: () throws -> [String] + ) throws { + guard exists else { + if afterCreate { throw posix(ENOENT) } + return // The caller uses O_EXCL (or the lookup syscall's ENOENT). + } + guard try list().contains(where: { $0.utf8.elementsEqual(name.utf8) }) else { + throw spellingMismatch(creating: creating, afterCreate: afterCreate) + } + } + + private static func spellingMismatch(creating: Bool, afterCreate: Bool) -> NSError { + guard afterCreate else { return posix(creating ? EEXIST : ENOENT) } + return NSError(domain: NSPOSIXErrorDomain, code: Int(EILSEQ), userInfo: [ + NSLocalizedDescriptionKey: + "This filesystem rewrites the requested name; use decomposed Unicode (NFD) on HFS+ or macOS exFAT.", + NSLocalizedRecoverySuggestionErrorKey: + "Use the exact spelling reported by directory listings.", + ]) + } + + private static func listNames(_ parent: Int32) throws -> [String] { + let fd = openat(parent, ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC) + guard fd >= 0 else { throw posix(errno) } + guard let stream = fdopendir(fd) else { + let error = errno + close(fd) + throw posix(error) + } + defer { closedir(stream) } + var names: [String] = [] + while true { + errno = 0 + guard let item = readdir(stream) else { + if errno != 0 { throw posix(errno) } + return names + } + let name = withUnsafePointer(to: &item.pointee.d_name) { + $0.withMemoryRebound(to: CChar.self, capacity: MemoryLayout.size(ofValue: item.pointee.d_name)) { + String(validatingCString: $0) + } + } + // Unrepresentable entries cannot byte-match a valid Swift String. + if let name { names.append(name) } + } + } + + private func posix(_ code: Int32) -> NSError { Self.posix(code) } + private static func posix(_ code: Int32) -> NSError { + NSError(domain: NSPOSIXErrorDomain, code: Int(code)) + } +} diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index cdb6388..eb4964b 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -96,10 +96,10 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst let mounts = mountsSnapshot() let syntheticNames = syntheticChildMountNames(under: path, mounts: mounts) let resolved = resolveWritable(path: path, mounts: mounts) - var merged: [String: DirectoryEntry] = [:] + var merged: [PathName: DirectoryEntry] = [:] do { for entry in try await resolved.filesystem.listDirectory(path: resolved.relativePath) { - merged[entry.name] = DirectoryEntry( + merged[PathName(entry.name)] = DirectoryEntry( name: entry.name, info: Self.virtualInfo(entry.info, at: path.appending(entry.name)) ) @@ -124,9 +124,9 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst let child = path.appending(name) // Actual mounts shadow entries. Synthetic ancestors use backend metadata if // present, keeping traversal consistent with stat (including regular files). - merged[name] = DirectoryEntry(name: name, info: try await stat(path: child)) + merged[PathName(name)] = DirectoryEntry(name: name, info: try await stat(path: child)) } - return merged.values.sorted { $0.name < $1.name } + return merged.values.sorted { PathName($0.name) < PathName($1.name) } } /// See ``FileSystem/readFile(path:)``. @@ -387,16 +387,16 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst } private func syntheticChildMountNames(under path: WorkspacePath, mounts: [Mount]) -> [String] { - var names = Set() + var names = Set() for mount in mounts where mount.mountPoint != path { guard isPath(mount.mountPoint, inside: path) else { continue } if let first = mount.mountPoint.components.dropFirst(path.components.count).first { - names.insert(first) + names.insert(PathName(first)) } } - return names.sorted() + return names.sorted().map(\.string) } private func isPath(_ candidate: WorkspacePath, inside parent: WorkspacePath) -> Bool { diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index 36a16a6..286554f 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -118,7 +118,7 @@ public actor OverlayFileSystem: FileSystem { throw posixError(ENOENT) } - var byName: [String: DirectoryEntry] = [:] + var byName: [PathName: DirectoryEntry] = [:] if let lowerInfo, upperInfo == nil || upperInfo?.kind == .directory { guard lowerInfo.kind == .directory || upperInfo != nil else { throw posixError(ENOTDIR) @@ -126,7 +126,7 @@ public actor OverlayFileSystem: FileSystem { if lowerInfo.kind == .directory, let lower { for entry in try await lower.listDirectory(path: path) where !isCut(path.appending(entry.name)) { - byName[entry.name] = entry + byName[PathName(entry.name)] = entry } } } @@ -135,14 +135,14 @@ public actor OverlayFileSystem: FileSystem { throw posixError(ENOTDIR) } for entry in try await upper.listDirectory(path: path) { - byName[entry.name] = entry + byName[PathName(entry.name)] = entry } } return byName.values.map { entry in var info = entry.info info.path = originalPath.appending(entry.name) return DirectoryEntry(name: entry.name, info: info) - }.sorted { $0.name < $1.name } + }.sorted { PathName($0.name) < PathName($1.name) } } /// See ``FileSystem/readFile(path:)``. diff --git a/Sources/Workspace/RevisionIndex.swift b/Sources/Workspace/RevisionIndex.swift index 3585c55..8dff252 100644 --- a/Sources/Workspace/RevisionIndex.swift +++ b/Sources/Workspace/RevisionIndex.swift @@ -7,6 +7,19 @@ struct RevisionIndex: Sendable, Equatable { case directory(path: WorkspacePath, permissions: POSIXPermissions, children: [Entry]) case symbolicLink(path: WorkspacePath, target: String, permissions: POSIXPermissions) + static func == (lhs: Self, rhs: Self) -> Bool { + switch (lhs, rhs) { + case let (.missing(a), .missing(b)): return a == b + case let (.file(a, sizeA, modeA, idA), .file(b, sizeB, modeB, idB)): + return a == b && sizeA == sizeB && modeA == modeB && idA == idB + case let (.directory(a, modeA, childrenA), .directory(b, modeB, childrenB)): + return a == b && modeA == modeB && childrenA == childrenB + case let (.symbolicLink(a, targetA, modeA), .symbolicLink(b, targetB, modeB)): + return a == b && modeA == modeB && targetA.utf8.elementsEqual(targetB.utf8) + default: return false + } + } + var path: WorkspacePath { switch self { case let .missing(path), let .file(path, _, _, _), let .directory(path, _, _), diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index 9a9aca1..f1719e4 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -83,6 +83,11 @@ struct Snapshot: Sendable, Codable, Equatable { /// The symbolic link's POSIX permissions at the time of capture. var permissions: POSIXPermissions + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.path == rhs.path && lhs.permissions == rhs.permissions + && lhs.target.utf8.elementsEqual(rhs.target.utf8) + } + /// Creates a symlink entry. init(path: WorkspacePath, target: String, permissions: POSIXPermissions) { self.path = path @@ -181,7 +186,7 @@ extension Snapshot { case .directory: let entries = try await target.listDirectory(path: path) let children = try await entries - .sorted { $0.name < $1.name } + .sorted { PathName($0.name) < PathName($1.name) } .snapshotAsyncMap { entry in try await captureEntry(from: target, at: path.appending(entry.name)) } @@ -233,7 +238,7 @@ extension Snapshot { var rewriteRequired = true if let info = try await infoIfPresent(at: symlink.path, on: target) { - if info.kind == .symlink, try await target.readSymlink(path: symlink.path) == symlink.target { + if info.kind == .symlink, try await target.readSymlink(path: symlink.path).utf8.elementsEqual(symlink.target.utf8) { rewriteRequired = false } else { try await removeRecursively(at: symlink.path, on: target) @@ -333,10 +338,10 @@ extension Snapshot { under parentPath: WorkspacePath, on target: any FileSystem ) async throws { - let expectedNames = Set(expectedChildren.map { $0.path.basename }) + let expectedNames = Set(expectedChildren.map { PathName($0.path.basename) }) let existingEntries = try await target.listDirectory(path: parentPath) - for entry in existingEntries where !expectedNames.contains(entry.name) { + for entry in existingEntries where !expectedNames.contains(PathName(entry.name)) { try await removeRecursively(at: parentPath.appending(entry.name), on: target) } diff --git a/Sources/Workspace/Support/FileChangeStreams.swift b/Sources/Workspace/Support/FileChangeStreams.swift index de02f91..5e67c74 100644 --- a/Sources/Workspace/Support/FileChangeStreams.swift +++ b/Sources/Workspace/Support/FileChangeStreams.swift @@ -53,7 +53,7 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { private let continuation: AsyncStream.Continuation private let descriptor: Int32 private let source: DispatchSourceRead - private var pathsByWatch: [Int32: Set] = [:] + private var pathsByWatch: [Int32: Set] = [:] // Keep the root's parent watched so recreating the root can restore coverage even // when it was absent during the IN_DELETE_SELF/IN_MOVE_SELF rescan. private var parentWatch: Int32? @@ -112,7 +112,7 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { received = true } else { let nameBytes = bytes[(offset + headerSize)..<(offset + recordSize)].prefix { $0 != 0 } - guard String(decoding: nameBytes, as: UTF8.self) == directory.lastPathComponent else { continue } + guard nameBytes.elementsEqual(directory.lastPathComponent.utf8) else { continue } } } received = true @@ -171,21 +171,21 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { IN_CREATE | IN_DELETE | IN_CLOSE_WRITE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF | IN_ONLYDIR ) - var nextByWatch: [Int32: Set] = [:] + var nextByWatch: [Int32: Set] = [:] for path in paths { // The kernel returns the current wd for the same inode and a new wd for a // replacement. Re-register every discovered path, rather than trusting its name. let watch = path.withCString { inotify_add_watch(descriptor, $0, mask) } if watch < 0 { let code = errno - if path == directory.path, requireRoot || (code != ENOENT && code != ENOTDIR) { + if path.utf8.elementsEqual(directory.path.utf8), requireRoot || (code != ENOENT && code != ENOTDIR) { throw POSIXError(.init(rawValue: code) ?? .EIO) } // Children can disappear, become files, or deny reads during enumeration. // Skip them and keep registering later siblings; future events will retry. continue } - nextByWatch[watch, default: []].insert(path) + nextByWatch[watch, default: []].insert(PathName(path)) } // A descriptor can serve multiple paths. Remove it only after every live path has // been registered, and only if the new reverse map no longer references that wd. diff --git a/Sources/Workspace/Support/PathName.swift b/Sources/Workspace/Support/PathName.swift new file mode 100644 index 0000000..9a60639 --- /dev/null +++ b/Sources/Workspace/Support/PathName.swift @@ -0,0 +1,20 @@ +/// Byte-exact keys for directory names and host paths. Swift String keys instead +/// merge canonically equivalent UTF-8 spellings, which may name different entries. +struct PathName: Sendable, Hashable, Comparable { + let string: String + + init(_ string: String) { self.string = string } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.string.utf8.elementsEqual(rhs.string.utf8) + } + + func hash(into hasher: inout Hasher) { + var contiguous = string + contiguous.withUTF8 { hasher.combine(bytes: UnsafeRawBufferPointer($0)) } + } + + static func < (lhs: Self, rhs: Self) -> Bool { + lhs.string.utf8.lexicographicallyPrecedes(rhs.string.utf8) + } +} diff --git a/Sources/Workspace/Support/WorkspacePath.swift b/Sources/Workspace/Support/WorkspacePath.swift index c6c12a0..5b8ffa3 100644 --- a/Sources/Workspace/Support/WorkspacePath.swift +++ b/Sources/Workspace/Support/WorkspacePath.swift @@ -4,6 +4,8 @@ import Foundation /// /// `WorkspacePath` models shell-style paths independently of the host filesystem so the same value can /// be used with in-memory, mounted, overlay, and disk-backed filesystems. +/// Identity, hashing, ordering, and containment use exact UTF-8 bytes. Case and Unicode +/// normalization are significant: `/café` and `/cafe\u{301}` are different paths. public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, ExpressibleByStringLiteral, CustomStringConvertible { @@ -70,25 +72,15 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl Self.splitComponents(storage) } - /// Whether this path is equal to or below `ancestor`, using canonical Unicode - /// equality for each component, just like path equality. + /// Whether this path is equal to or below `ancestor`, comparing exact UTF-8 + /// component bytes, just like WorkspacePath's `==`. func isWithin(_ ancestor: WorkspacePath) -> Bool { if ancestor.isRoot { return true } - - // The fast path stays allocation-free: UTF-8 views and slices avoid - // constructing component arrays or new strings in containment hot loops. let bytes = storage.utf8 let ancestorBytes = ancestor.storage.utf8 - if bytes.starts(with: ancestorBytes) { - let next = bytes.dropFirst(ancestorBytes.count).first - if next == nil || next == 47 { return true } - } - if bytes.allSatisfy({ $0 < 128 }), ancestorBytes.allSatisfy({ $0 < 128 }) { - return false - } - - // Different non-ASCII byte spellings may still be canonically equal. - return components.starts(with: ancestor.components) + guard bytes.starts(with: ancestorBytes) else { return false } + let next = bytes.dropFirst(ancestorBytes.count).first + return next == nil || next == 47 } /// The final path component, or `/` for the root path. @@ -112,9 +104,17 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl Self.join(self, String(component)) } - /// Orders paths lexicographically by their normalized string values. + public static func == (lhs: WorkspacePath, rhs: WorkspacePath) -> Bool { + lhs.storage.utf8.elementsEqual(rhs.storage.utf8) + } + + public func hash(into hasher: inout Hasher) { + PathName(storage).hash(into: &hasher) + } + + /// Orders paths lexicographically by their normalized UTF-8 bytes. public static func < (lhs: WorkspacePath, rhs: WorkspacePath) -> Bool { - lhs.storage < rhs.storage + lhs.storage.utf8.lexicographicallyPrecedes(rhs.storage.utf8) } /// Decodes a path from its string representation. diff --git a/Sources/Workspace/Workspace+Merge.swift b/Sources/Workspace/Workspace+Merge.swift index 9e0c7cf..d9c8de7 100644 --- a/Sources/Workspace/Workspace+Merge.swift +++ b/Sources/Workspace/Workspace+Merge.swift @@ -7,6 +7,11 @@ extension Workspace { var permissions: POSIXPermissions var fileData: Data? var symlinkTarget: String? + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.kind == rhs.kind && lhs.permissions == rhs.permissions && lhs.fileData == rhs.fileData + && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) + } } static func flattenMergeNodes(_ entry: Snapshot.Entry, into nodes: inout [WorkspacePath: MergeNode]) { @@ -82,7 +87,7 @@ extension Workspace { .init( path: path, permissions: node.permissions, - children: (childNames[path] ?? []).sorted { $0.0 < $1.0 }.map { name, child in + children: (childNames[path] ?? []).sorted { PathName($0.0) < PathName($1.0) }.map { name, child in subtree(at: path.appending(name), node: child) } ) diff --git a/Sources/Workspace/WorkspaceArchive.swift b/Sources/Workspace/WorkspaceArchive.swift index 4ef7c16..17ce946 100644 --- a/Sources/Workspace/WorkspaceArchive.swift +++ b/Sources/Workspace/WorkspaceArchive.swift @@ -9,6 +9,19 @@ extension Workspace { case directory(path: WorkspacePath, permissions: POSIXPermissions, children: [Entry]) case symbolicLink(path: WorkspacePath, target: String, permissions: POSIXPermissions) + public static func == (lhs: Self, rhs: Self) -> Bool { + switch (lhs, rhs) { + case let (.missing(a), .missing(b)): return a == b + case let (.file(a, dataA, modeA), .file(b, dataB, modeB)): + return a == b && dataA == dataB && modeA == modeB + case let (.directory(a, modeA, childrenA), .directory(b, modeB, childrenB)): + return a == b && modeA == modeB && childrenA == childrenB + case let (.symbolicLink(a, targetA, modeA), .symbolicLink(b, targetB, modeB)): + return a == b && modeA == modeB && targetA.utf8.elementsEqual(targetB.utf8) + default: return false + } + } + public var path: WorkspacePath { switch self { case let .missing(path), let .file(path, _, _), let .directory(path, _, _), @@ -36,12 +49,12 @@ extension Workspace { var pending = [entry] while let entry = pending.popLast() { guard case let .directory(parent, _, children) = entry else { continue } - var names = Set() + var names = Set() for child in children { guard child.path != parent, child.path.parent == parent else { throw WorkspaceError.unsupported("invalid archive: \(child.path) is not an immediate child of \(parent)") } - guard names.insert(child.path.name).inserted else { + guard names.insert(PathName(child.path.name)).inserted else { throw WorkspaceError.unsupported("invalid archive: duplicate child \(child.path.name) in \(parent)") } pending.append(child) diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index de1cf94..5507655 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -179,7 +179,7 @@ extension Workspace { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) } return children.map { DirectoryEntry(name: $0.path.name, info: Self.info(for: $0)) } - .sorted { $0.name < $1.name } + .sorted { PathName($0.name) < PathName($1.name) } } } diff --git a/Sources/WorkspacePOSIX/WorkspacePOSIX.c b/Sources/WorkspacePOSIX/WorkspacePOSIX.c new file mode 100644 index 0000000..7d28b07 --- /dev/null +++ b/Sources/WorkspacePOSIX/WorkspacePOSIX.c @@ -0,0 +1,32 @@ +#include "WorkspacePOSIX.h" +#include +#include +#include +#include + +// Swift Glibc omits statfs and the architecture-dependent ioctl request macros. +int workspace_filesystem_type(int fd, uint64_t *type) { + struct statfs info; + if (fstatfs(fd, &info) < 0) return -1; + *type = (uint64_t)info.f_type; + return 0; +} + +int workspace_directory_flags(int fd, uint32_t *flags) { + int value = 0; + if (ioctl(fd, FS_IOC_GETFLAGS, &value) < 0) return -1; + *flags = (uint32_t)value; + return 0; +} + +int workspace_xfs_case_insensitive(int fd, int *insensitive) { + // XFS_IOC_FSGEOMETRY's stable v5 ABI: 256 bytes, uint32_t flags at offset 92. + // Use the system ioctl encoding without requiring the optional libxfs headers. + // https://github.com/torvalds/linux/blob/master/fs/xfs/libxfs/xfs_fs.h + struct { uint64_t words[32]; } geometry = {0}; + if (ioctl(fd, _IOR('X', 126, geometry), &geometry) < 0) return -1; + uint32_t flags; + memcpy(&flags, (const unsigned char *)&geometry + 92, sizeof(flags)); + *insensitive = (flags & (1U << 12)) != 0; // XFS_FSOP_GEOM_FLAGS_DIRV2CI + return 0; +} diff --git a/Sources/WorkspacePOSIX/include/WorkspacePOSIX.h b/Sources/WorkspacePOSIX/include/WorkspacePOSIX.h new file mode 100644 index 0000000..2d95949 --- /dev/null +++ b/Sources/WorkspacePOSIX/include/WorkspacePOSIX.h @@ -0,0 +1,7 @@ +#ifndef WORKSPACE_POSIX_H +#define WORKSPACE_POSIX_H +#include +int workspace_filesystem_type(int fd, uint64_t *type); +int workspace_directory_flags(int fd, uint32_t *flags); +int workspace_xfs_case_insensitive(int fd, int *insensitive); +#endif diff --git a/Tests/WorkspaceTests/LocalPathSpellingTests.swift b/Tests/WorkspaceTests/LocalPathSpellingTests.swift new file mode 100644 index 0000000..eeb3c75 --- /dev/null +++ b/Tests/WorkspaceTests/LocalPathSpellingTests.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing +@testable import Workspace +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +@Suite("Stored path spelling") +struct LocalPathSpellingTests { + @Test(arguments: [("secret", "SECRET"), ("caf\u{e9}", "cafe\u{301}")]) + func listingOverridesPrimedLookupAliases(stored: String, alias: String) throws { + // Model a filesystem whose kernel lookup/name cache accepts the alias + // (and may even report it as the attribute name), while readdir is exact. + let listing = { [stored] } + try LocalPathSpelling.checkListing(name: stored, exists: true, creating: false, list: listing) + #expect(throws: POSIXError(.ENOENT) as NSError) { + try LocalPathSpelling.checkListing(name: alias, exists: true, creating: false, list: listing) + } + #expect(throws: POSIXError(.EEXIST) as NSError) { + try LocalPathSpelling.checkListing(name: alias, exists: true, creating: true, list: listing) + } + do { + try LocalPathSpelling.checkListing( + name: alias, exists: true, creating: true, afterCreate: true, list: listing) + Issue.record("expected a rewritten creation to fail") + } catch { + let error = error as NSError + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(EILSEQ)) + #expect(error.localizedDescription.contains("NFD")) + #expect(error.localizedRecoverySuggestion?.contains("directory listings") == true) + } + } + + @Test + func listingIsFreshEvenWhenDirectoryTimestampsAreUnchanged() throws { + var names = ["secret"] + let list = { names } + try LocalPathSpelling.checkListing(name: "secret", exists: true, creating: false, list: list) + // FAT-like timestamp granularity cannot be used to reuse the first result. + names = ["SECRET"] + #expect(throws: POSIXError(.ENOENT) as NSError) { + try LocalPathSpelling.checkListing(name: "secret", exists: true, creating: false, list: list) + } + try LocalPathSpelling.checkListing(name: "SECRET", exists: true, creating: false, list: list) + try LocalPathSpelling.checkListing(name: "new", exists: false, creating: true) { + throw POSIXError(.EACCES) // Missing creations rely on O_EXCL, without enumeration. + } + } + + @Test + func listingPolicyUsesStoredNamesFromRealDirectory() throws { + let root = try TestSupport.temporaryDirectory("SpellingListing") + defer { TestSupport.remove(root) } + let parent = open(root.path, O_RDONLY | O_DIRECTORY) + #expect(parent >= 0) + defer { close(parent) } + let file = openat(parent, "secret", O_CREAT | O_EXCL | O_WRONLY, 0o600) + #expect(file >= 0) + close(file) + // Prime the kernel name cache before checking, if the volume aliases case. + let alias = openat(parent, "SECRET", O_RDONLY) + if alias >= 0 { close(alias) } + let checker = LocalPathSpelling() + try checker.check(parent: parent, name: "secret", policy: .verifyByListing, creating: false) + if alias >= 0 { + #expect(throws: POSIXError(.ENOENT) as NSError) { + try checker.check(parent: parent, name: "SECRET", policy: .verifyByListing, creating: false) + } + #expect(throws: POSIXError(.EEXIST) as NSError) { + try checker.check(parent: parent, name: "SECRET", policy: .verifyByListing, creating: true) + } + } + } +} diff --git a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift new file mode 100644 index 0000000..4870a96 --- /dev/null +++ b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift @@ -0,0 +1,363 @@ +import Foundation +import Testing +@testable import Workspace +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +@Suite("Exact path identity") +struct PathIdentityRegressionTests { + private let nfc: WorkspacePath = "/caf\u{e9}" + private let nfd: WorkspacePath = "/cafe\u{301}" + + // Foundation URL file writes can normalize the last component on macOS. + private func seed(_ root: URL, _ name: String, data: Data = Data([1, 2])) throws { + let fd = open(root.path + "/" + name, O_WRONLY | O_CREAT | O_EXCL, 0o600) + guard fd >= 0 else { throw POSIXError(.init(rawValue: errno) ?? .EIO) } + defer { close(fd) } + let written = data.withUnsafeBytes { write(fd, $0.baseAddress!, $0.count) } + guard written == data.count else { throw POSIXError(.EIO) } + } + + private func expectPOSIX(_ code: Int32, _ operation: () async throws -> Void) async { + do { + try await operation() + Issue.record("expected POSIX error \(code), operation succeeded") + } catch { + let error = error as NSError + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(code)) + } + } + + @Test + func pathsAndDirectoryKeysKeepCanonicalTwinsDistinct() async throws { + #expect(nfc != nfd) + #expect(Set([nfc, nfd]).count == 2) + #expect([nfc, nfd].sorted() == [nfd, nfc]) + #expect(!nfd.appending("x").isWithin(nfc)) + let memory = InMemoryFileSystem() + for (path, data) in [(nfc, Data([1])), (nfd, Data([2]))] { + try await memory.createFile(path: path, data: data) + } + #expect(try await memory.listDirectory(path: .root).count == 2) + #expect(try await memory.readFile(path: nfc) == Data([1])) + #expect(try await memory.readFile(path: nfd) == Data([2])) + let mounted = MountedFileSystem(base: memory) + #expect(try await mounted.listDirectory(path: .root).count == 2) + let overlay = OverlayFileSystem(over: memory) + #expect(try await overlay.listDirectory(path: .root).count == 2) + try await overlay.remove(path: nfc, recursive: false) + #expect(try await overlay.readFile(path: nfd) == Data([2])) + } + + @Test(arguments: [false, true]) + func allowRulesDoNotGrantCanonicalTwin(mounted: Bool) async throws { + let memory = InMemoryFileSystem() + let base: any FileSystem = mounted + ? MountedFileSystem(mounts: [.init(mountPoint: nfd, fileSystem: memory)]) : memory + try await base.createDirectory(path: nfd, recursive: true) + try await base.writeFile(path: nfd.appending("secret"), data: Data([42]), append: false) + let fs = AuthorizedFileSystem(base: base, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.readFile], pathPrefix: nfc, effect: .allow), + ])) + await #expect(throws: WorkspaceError.self) { _ = try await fs.readFile(path: nfd.appending("secret")) } + } + + @Test + func quotasCountBothCanonicalTwins() async throws { + // Raw disk fixtures let the old accounting dictionary demonstrate its own + // collapse, independently of the old in-memory directory dictionary. + #if canImport(Glibc) + let root = try TestSupport.temporaryDirectory("IdentityQuota") + defer { TestSupport.remove(root) } + try seed(root, nfc.name) + try seed(root, nfd.name) + let base: any FileSystem = try LocalFileSystem(root: root) + #else + let base: any FileSystem = InMemoryFileSystem() + try await base.writeFile(path: nfc, data: Data([1, 2]), append: false) + try await base.writeFile(path: nfd, data: Data([3, 4]), append: false) + #endif + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 4)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 5, limit: 4)) { + try await fs.writeFile(path: "/extra", data: Data([5]), append: false) + } + #expect(!(await base.exists(path: "/extra"))) + } + + @Test(arguments: [false, true]) + func checkpointsArchivesAndDiffsKeepBothNames(diskStore: Bool) async throws { + let root = try TestSupport.temporaryDirectory("IdentityHistory") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: diskStore ? .directory(root) : .memory) + try await workspace.writeData(Data([1]), to: nfc) + try await workspace.writeData(Data([2]), to: nfd) + let before = try await workspace.createCheckpoint() + #expect(try await workspace.readData(from: nfc, at: .checkpoint(before.id)) == Data([1])) + #expect(try await workspace.readData(from: nfd, at: .checkpoint(before.id)) == Data([2])) + let archive = try await workspace.archive(from: .checkpoint(before.id)) + let restored = Workspace() + try await restored.restore(archive) + #expect(try await restored.list(.root).count == 2) + try await workspace.writeData(Data([3]), to: nfd) + let after = try await workspace.createCheckpoint() + let diff = try await workspace.diff(from: .checkpoint(before.id), to: .checkpoint(after.id)) + #expect(diff.touchedPaths.count == 1) + #expect(diff.touchedPaths.first?.string.utf8.elementsEqual(nfd.string.utf8) == true) + try await workspace.remove(nfd) + try await restored.restore(workspace.archive()) + let remaining = try await restored.list(.root) + #expect(remaining.count == 1) + #expect(remaining.first?.info.path.string.utf8.elementsEqual(nfc.string.utf8) == true) + } + + @Test + func parentedCanonicalTwinSavesInMemory() async { + await #expect(processExitsWith: .success) { + try await PathIdentityRegressionTests.checkParentedCanonicalTwinSaves(diskStore: false) + } + } + + @Test + func parentedCanonicalTwinSavesInDirectory() async { + await #expect(processExitsWith: .success) { + try await PathIdentityRegressionTests.checkParentedCanonicalTwinSaves(diskStore: true) + } + } + + private static func checkParentedCanonicalTwinSaves(diskStore: Bool) async throws { + let root = try TestSupport.temporaryDirectory("IdentityParentedSaves") + defer { TestSupport.remove(root) } + let store: any CheckpointStore = diskStore + ? FileCheckpointStore(rootDirectory: root) : InMemoryCheckpointStore() + let workspaceID = UUID() + try await store.prepare(workspaceId: workspaceID) + let nfc: WorkspacePath = "/caf\u{e9}" + let nfd: WorkspacePath = "/cafe\u{301}" + // Supply snapshots directly: the old InMemoryFileSystem collapses these + // names before capture and would mask the checkpoint-store crash. + let contents: [[(WorkspacePath, UInt8)]] = [ + [(nfd, 1), (nfc, 2)], + [(nfd, 3), (nfc, 4)], + [(nfd, 5)], + ] + var checkpoints: [Checkpoint] = [] + for files in contents { + let snapshot = Snapshot(rootPath: .root, entry: .directory(.init( + path: .root, permissions: .defaultDirectory, + children: files.map { path, byte in + .file(.init(path: path, data: Data([byte]), permissions: .defaultFile)) + } + ))) + let parentID = checkpoints.last?.id + let checkpoint = try await store.saveRevision(snapshot, draft: .init( + workspaceID: workspaceID, label: nil, preferredParentID: parentID, origin: .manual + )) + #expect(checkpoint.parentID == parentID) + checkpoints.append(checkpoint) + } + + // Reopen directory persistence so the assertions exercise serialized + // manifests and blobs, as well as the in-memory store's materialized tree. + let reader: any CheckpointStore = diskStore + ? FileCheckpointStore(rootDirectory: root) : store + let restored = InMemoryFileSystem() + var previousSnapshot: Snapshot? + var previousIndex: RevisionIndex? + for (offset, checkpoint) in checkpoints.enumerated() { + let snapshot = try #require(await reader.loadSnapshot( + id: checkpoint.snapshotId, workspaceId: workspaceID + )) + let index = try #require(await reader.loadRevisionIndex( + id: checkpoint.snapshotId, workspaceId: workspaceID + )) + if let previousSnapshot, let previousIndex { + let snapshotDiff = ChangeSet.compare( + before: previousSnapshot, after: snapshot, maxTextBytes: 0 + ) + let indexDiff = try await ChangeSet.compare( + before: previousIndex, after: index, maxTextBytes: 0, + loadBefore: { _ in throw POSIXError(.EIO) }, + loadAfter: { _ in throw POSIXError(.EIO) } + ) + // Compare bytes explicitly so baseline String equality cannot + // conceal a dropped twin or incorrect ordering. + let expected = [nfd, nfc].map { Array($0.string.utf8) } + #expect(snapshotDiff.changes.map { Array($0.path.string.utf8) } == expected) + #expect(indexDiff.changes.map { Array($0.path.string.utf8) } == expected) + } + try await Snapshot.restore(snapshot, to: restored) + let entries = try await restored.listDirectory(path: .root) + #expect(entries.map { Array($0.name.utf8) } == contents[offset].map { Array($0.0.name.utf8) }) + for (path, byte) in contents[offset] { + #expect(try await restored.readFile(path: path) == Data([byte])) + } + previousSnapshot = snapshot + previousIndex = index + } + } + + @Test + func symlinkTargetChangesAreByteExact() async throws { + let workspace = Workspace() + try await workspace.createSymbolicLink("/alias", target: nfc.string) + let before = try await workspace.createCheckpoint() + try await workspace.remove("/alias") + let changed = try await workspace.createSymbolicLink("/alias", target: nfd.string) + #expect(changed.touchedPaths == ["/alias"]) + let after = try await workspace.createCheckpoint() + #expect(try await workspace.diff(from: .checkpoint(before.id), to: .checkpoint(after.id)).touchedPaths == ["/alias"]) + } + + #if canImport(Darwin) + private func caseInsensitive(_ root: URL) throws -> Bool { + try seed(root, "case-probe") + let insensitive = access(root.path + "/CASE-PROBE", F_OK) == 0 + if !insensitive { print("SKIP: exact-spelling case tests require a case-insensitive volume: \(root.path)") } + return insensitive + } + + @Test + func caseAliasCannotBypassDenyRule() async throws { + let root = try TestSupport.temporaryDirectory("IdentityAuthorization") + defer { TestSupport.remove(root) } + guard try caseInsensitive(root) else { return } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/private", recursive: true) + try await local.writeFile(path: "/private/secret", data: Data([42]), append: false) + let fs = AuthorizedFileSystem(base: local, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.readFile], pathPrefix: "/private", effect: .deny(message: nil)), + ], defaultEffect: .allow)) + await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: "/PRIVATE/secret") } + try await local.createSymlink(path: "/alias", target: "private/secret") + #expect(try await local.resolveRealPath(path: "/alias") == "/private/secret") + await #expect(throws: WorkspaceError.self) { _ = try await fs.readFile(path: "/alias") } + } + + @Test + func caseVariantOperationsRejectAliases() async throws { + let root = try TestSupport.temporaryDirectory("IdentityCase") + defer { TestSupport.remove(root) } + guard try caseInsensitive(root) else { return } + let fs = try LocalFileSystem(root: root) + try await fs.writeFile(path: "/file", data: Data([7]), append: false) + await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: "/FILE") } + await expectPOSIX(ENOENT) { _ = try await fs.stat(path: "/FILE") } + await expectPOSIX(ENOENT) { _ = try await fs.resolveRealPath(path: "/FILE") } + await expectPOSIX(EEXIST) { try await fs.writeFile(path: "/FILE", data: Data([9]), append: false) } + await expectPOSIX(EEXIST) { try await fs.writeFile(path: "/FILE", data: Data([9]), append: true) } + await expectPOSIX(EEXIST) { try await fs.createFile(path: "/FILE", data: Data([9])) } + await expectPOSIX(EEXIST) { try await fs.move(from: "/file", to: "/FILE") } + await expectPOSIX(ENOENT) { try await fs.move(from: "/FILE", to: "/moved") } + try await fs.remove(path: "/FILE", recursive: false) // missing removal is a no-op + #expect(try await fs.readFile(path: "/file") == Data([7])) + try await fs.createDirectory(path: "/dir", recursive: false) + await expectPOSIX(EEXIST) { try await fs.createDirectory(path: "/DIR", recursive: true) } + await expectPOSIX(EEXIST) { try await fs.writeFile(path: "/DIR/new", data: Data([1]), append: false) } + await expectPOSIX(EEXIST) { try await fs.copy(from: "/file", to: "/FILE", recursive: false) } + await expectPOSIX(ENOENT) { try await fs.createHardLink(path: "/link", target: "/FILE") } + #expect(try await fs.listDirectory(path: "/dir").isEmpty) + } + + @Test + func middleAncestorAliasIsRejectedAcrossOpenPaths() async throws { + let root = try TestSupport.temporaryDirectory("IdentityAncestors") + defer { TestSupport.remove(root) } + guard try caseInsensitive(root) else { return } + let fs = try LocalFileSystem(root: root) + try await fs.createFile(path: "/one/middle/three/file", data: Data([42])) + // Prime the alias before testing both metadata and open-based operations. + let aliasFD = open(root.path + "/one/MIDDLE/three/file", O_RDONLY) + #expect(aliasFD >= 0) + close(aliasFD) + await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: "/one/MIDDLE/three/file") } + await expectPOSIX(ENOENT) { _ = try await fs.stat(path: "/one/MIDDLE/three/file") } + await expectPOSIX(ENOENT) { _ = try await fs.listDirectory(path: "/one/MIDDLE/three") } + await expectPOSIX(ENOENT) { _ = try await fs.resolveRealPath(path: "/one/MIDDLE/three/file") } + await expectPOSIX(EEXIST) { + try await fs.writeFile(path: "/one/MIDDLE/three/new", data: Data([9]), append: false) + } + try await fs.createSymlink(path: "/relative", target: "one/MIDDLE/../middle/three/file") + await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: "/relative") } + #expect(try await fs.readFile(path: "/one/middle/three/file") == Data([42])) + #expect(!(await fs.exists(path: "/one/middle/three/new"))) + } + + @Test + func longAPFSPathsFallBackWithoutConfusingHardLinkNames() async throws { + let root = try TestSupport.temporaryDirectory("IdentityLongPath") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + let component = String(repeating: "d", count: 180) + let parent = try WorkspacePath("/" + Array(repeating: component, count: 7).joined(separator: "/")) + let path = parent.appending("file") + try await fs.createFile(path: path, data: Data([42])) + try await fs.createHardLink(path: "/other-name", target: path) + // Opening the other hard link must not make F_GETPATH on a final file fd + // an apparent spelling mismatch. Directory paths also exceed MAXPATHLEN. + #expect(try await fs.readFile(path: "/other-name") == Data([42])) + #expect(try await fs.readFile(path: path) == Data([42])) + #expect(try await fs.resolveRealPath(path: path) == path) + try await fs.remove(path: .root.appending(component), recursive: true) + #expect(try await fs.readFile(path: "/other-name") == Data([42])) + } + + @Test + func apfsRejectsNormalizationAliases() async throws { + let root = try TestSupport.temporaryDirectory("IdentityNormalization") + defer { TestSupport.remove(root) } + try seed(root, nfc.name) + guard access(root.path + nfd.string, F_OK) == 0 else { + print("SKIP: normalization-alias test requires a normalization-insensitive volume") + return + } + let fs = try LocalFileSystem(root: root) + #expect(try await fs.readFile(path: nfc) == Data([1, 2])) + await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: nfd) } + await expectPOSIX(EEXIST) { try await fs.writeFile(path: nfd, data: Data([9]), append: false) } + await expectPOSIX(EEXIST) { try await fs.createFile(path: nfd, data: Data([9])) } + #expect(try await fs.resolveRealPath(path: nfc).string.utf8.elementsEqual(nfc.string.utf8)) + #expect(try await fs.listDirectory(path: .root).map(\.name).first?.utf8.elementsEqual(nfc.name.utf8) == true) + try await fs.remove(path: nfc, recursive: false) + try await fs.createFile(path: nfd, data: Data([3])) + #expect(try await fs.readFile(path: nfd) == Data([3])) + #expect(try await fs.listDirectory(path: .root).first?.name.utf8.elementsEqual(nfd.name.utf8) == true) + await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: nfc) } + } + #endif + + #if canImport(Glibc) + @Test + func linuxRejectsUndecodableDirectoryNames() async throws { + let root = try TestSupport.temporaryDirectory("IdentityInvalidUTF8") + defer { TestSupport.remove(root) } + let bytes = Array((root.path + "/").utf8) + [0xFF, 0] + let fd = bytes.withUnsafeBytes { + open($0.baseAddress!.assumingMemoryBound(to: CChar.self), O_WRONLY | O_CREAT | O_EXCL, 0o600) + } + guard fd >= 0 else { throw POSIXError(.init(rawValue: errno) ?? .EIO) } + close(fd) + try seed(root, "\u{fffd}") + let fs = try LocalFileSystem(root: root) + await #expect(throws: WorkspaceError.self) { _ = try await fs.listDirectory(path: .root) } + } + + @Test + func linuxLocalAllowRuleDoesNotGrantCanonicalTwin() async throws { + let root = try TestSupport.temporaryDirectory("IdentityLinux") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: nfc, recursive: true) + try await local.createDirectory(path: nfd, recursive: true) + try await local.writeFile(path: nfd.appending("secret"), data: Data([42]), append: false) + let fs = AuthorizedFileSystem(base: local, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.readFile], pathPrefix: nfc, effect: .allow), + ])) + await #expect(throws: WorkspaceError.self) { _ = try await fs.readFile(path: nfd.appending("secret")) } + #expect(try await local.listDirectory(path: .root).count == 2) + } + #endif +} diff --git a/Tests/WorkspaceTests/PathResolutionTests.swift b/Tests/WorkspaceTests/PathResolutionTests.swift index f434834..486068f 100644 --- a/Tests/WorkspaceTests/PathResolutionTests.swift +++ b/Tests/WorkspaceTests/PathResolutionTests.swift @@ -105,6 +105,37 @@ struct PathResolutionTests { #expect(try Data(contentsOf: outside.appendingPathComponent("file")) == Data("outside".utf8)) } + @Test + func nestedPathsSurviveRootRelocation() async throws { + let container = try TestSupport.temporaryDirectory("RelocatedRoot") + defer { TestSupport.remove(container) } + let parent = container.appendingPathComponent("parent") + let root = parent.appendingPathComponent("root") + let fs = try LocalFileSystem(root: root) + try await fs.createFile(path: "/top", data: Data([1])) + try await fs.createFile(path: "/d/f", data: Data([2])) + + let renamed = parent.appendingPathComponent("renamed") + try FileManager.default.moveItem(at: root, to: renamed) + let movedParent = container.appendingPathComponent("moved-parent") + // Exercise both root and ancestor relocation on the same pinned fd. + for moveAncestor in [false, true] { + if moveAncestor { + try FileManager.default.moveItem(at: parent, to: movedParent) + } + #expect(try await fs.readFile(path: "/top") == Data([1])) + await #expect(throws: Never.self) { + let data = try await fs.readFile(path: "/d/f") + #expect(data == Data([2])) + } + await #expect(throws: Never.self) { + try await fs.writeFile(path: "/d/g", data: Data([3]), append: false) + #expect(try await fs.readFile(path: "/d/g") == Data([3])) + try await fs.remove(path: "/d/g", recursive: false) + } + } + } + @Test func realPathsUseCanonicalRoot() async throws { let container = try TestSupport.temporaryDirectory("RootAlias") @@ -421,9 +452,10 @@ struct PathResolutionTests { } } - // Search-only ancestors are a distinct permission contract from directory listing. + // Search-only traversal works with a cached byte-exact policy. Uncached Linux + // flag inspection or listing verification requires read access. @Test(.enabled(if: geteuid() != 0, "Root bypasses permission checks")) - func localOperationsTraverseSearchOnlyAncestors() async throws { + func localOperationsRespectSearchOnlyAncestorPolicy() async throws { let root = try TestSupport.temporaryDirectory("SearchOnly") let ancestor = root.appendingPathComponent("ancestor") defer { @@ -434,11 +466,25 @@ struct PathResolutionTests { try await fs.createDirectory(path: "/ancestor/writable", recursive: true) try await fs.createFile(path: "/ancestor/writable/file", data: Data("inside".utf8)) #expect(chmod(ancestor.path, 0o111) == 0) + #if canImport(Glibc) + do { + _ = try await fs.readFile(path: "/ancestor/writable/file") + } catch let error as NSError where error.domain == NSPOSIXErrorDomain && error.code == Int(EACCES) { + // FS_IOC_GETFLAGS cannot use O_PATH. If flags are not cached, listing + // verification must reject mutations too until read access is available. + await #expect(throws: POSIXError(.EACCES) as NSError) { + try await fs.createFile(path: "/ancestor/writable/new", data: Data()) + } + #expect(!FileManager.default.fileExists(atPath: ancestor.path + "/writable/new")) + #expect(chmod(ancestor.path, 0o755) == 0) + } + #endif #expect(try await fs.readFile(path: "/ancestor/writable/file") == Data("inside".utf8)) try await fs.createFile(path: "/ancestor/writable/new", data: Data()) try await fs.move(from: "/ancestor/writable/new", to: "/ancestor/writable/moved") try await fs.remove(path: "/ancestor/writable/moved", recursive: false) #expect(try await fs.listDirectory(path: "/ancestor/writable").map(\.name) == ["file"]) + #expect(chmod(ancestor.path, 0o111) == 0) await #expect(throws: (any Error).self) { _ = try await fs.listDirectory(path: "/ancestor") } diff --git a/Tests/WorkspaceTests/UnicodePathRegressionTests.swift b/Tests/WorkspaceTests/UnicodePathRegressionTests.swift index 3ee17bd..97f5e2c 100644 --- a/Tests/WorkspaceTests/UnicodePathRegressionTests.swift +++ b/Tests/WorkspaceTests/UnicodePathRegressionTests.swift @@ -93,12 +93,12 @@ struct UnicodePathRegressionTests { } @Test - func containmentFastPathPreservesCanonicalEqualityAndBoundaries() { - // These canonically equivalent spellings must reach the non-ASCII fallback. + func containmentUsesExactBytesAndBoundaries() { + // Canonically equivalent spellings are deliberately distinct workspace paths. let decomposed = WorkspacePath("/cafe\u{301}/x") let composed = WorkspacePath("/caf\u{e9}") - #expect(decomposed.isWithin(composed)) - #expect(composed.isWithin(WorkspacePath("/cafe\u{301}"))) + #expect(!decomposed.isWithin(composed)) + #expect(!composed.isWithin(WorkspacePath("/cafe\u{301}"))) #expect(!(WorkspacePath("/cafe\u{301}-other/x")).isWithin(composed)) #expect(!(WorkspacePath("/caf\u{e9}/x")).isWithin(WorkspacePath("/cafe"))) @@ -112,12 +112,12 @@ struct UnicodePathRegressionTests { } @Test - func denyRulesUseComponentBoundariesAndCanonicalEquality() async throws { + func denyRulesUseComponentBoundariesAndExactEquality() async throws { let authorizer = RuleBasedPermissionAuthorizer(rules: [ .init(operations: [.readFile], pathPrefix: "/private", effect: .deny(message: nil)), .init(operations: [.readFile], pathPrefix: "/caf\u{e9}", effect: .deny(message: nil)), ], defaultEffect: .allow) - for path: WorkspacePath in ["/private/\u{301}x", "/cafe\u{301}/\u{301}x", "/caf\u{e9}"] { + for path: WorkspacePath in ["/private/\u{301}x", "/caf\u{e9}/\u{301}x", "/caf\u{e9}"] { let decision = await authorizer.authorize(.init(operation: .readFile, path: path)) guard case .deny = decision else { Issue.record("deny rule did not match \(path)") From 6b8c232752b6f75dc83b958cc6c2f39e816a6ed4 Mon Sep 17 00:00:00 2001 From: Zac White Date: Fri, 9 Oct 2026 19:39:57 -0700 Subject: [PATCH 13/34] Add a release-mode benchmark harness for workspace operations WorkspaceBenchmarks (swift run -c release WorkspaceBenchmarks) generates deterministic small/medium/large fixtures and runs each scenario in a fresh child process against LocalFileSystem and InMemoryFileSystem with memory and directory persistence. It records wall time, exact FileSystem-boundary I/O counts from a counting wrapper, checkpoint-store bytes written, and memory (setup high-water, per-operation increase, and held RSS), writes JSON, and compares against a baseline with --compare. The potential-path resolver protocol and its witnesses move from internal to package access so the counting wrapper can forward it without changing LimitedFileSystem's resolution path; the public API is unchanged. --- .gitignore | 2 + Benchmarks/WorkspaceBenchmarks/CLI.swift | 401 ++++++++++++++++++ .../CountingFileSystem.swift | 153 +++++++ Benchmarks/WorkspaceBenchmarks/Fixture.swift | 191 +++++++++ .../WorkspaceBenchmarks/Measurement.swift | 124 ++++++ .../WorkspaceBenchmarks/Operations.swift | 103 +++++ .../WorkspaceBenchmarks/Scenarios.swift | 342 +++++++++++++++ .../WorkspaceBenchmarks/StoreFootprint.swift | 58 +++ .../TemporaryCleanup.swift | 82 ++++ Package.swift | 5 + README.md | 53 +++ Sources/Workspace/FS/FileSystem.swift | 2 +- Sources/Workspace/FS/InMemoryFileSystem.swift | 2 +- Sources/Workspace/FS/LimitedFileSystem.swift | 2 +- Sources/Workspace/FS/LocalFileSystem.swift | 2 +- Sources/Workspace/FS/MountedFileSystem.swift | 2 +- Sources/Workspace/FS/SandboxFileSystem.swift | 2 +- .../FS/SecurityScopedFileSystem.swift | 2 +- 18 files changed, 1521 insertions(+), 7 deletions(-) create mode 100644 Benchmarks/WorkspaceBenchmarks/CLI.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/Fixture.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/Measurement.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/Operations.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/Scenarios.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/StoreFootprint.swift create mode 100644 Benchmarks/WorkspaceBenchmarks/TemporaryCleanup.swift diff --git a/.gitignore b/.gitignore index c08a29b..90a613a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ references Package.resolved .DS_Store .cursor + +Benchmarks/Results/ diff --git a/Benchmarks/WorkspaceBenchmarks/CLI.swift b/Benchmarks/WorkspaceBenchmarks/CLI.swift new file mode 100644 index 0000000..7cee669 --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/CLI.swift @@ -0,0 +1,401 @@ +#if os(macOS) || os(Linux) +import Foundation +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif + +struct Options { + var preset = "small" + var fixtureDirectory: URL? + var json: URL? + var compare: URL? + var filter: String? + var iterations = 5 + var warmups = 1 + var watcherTimeout = 15.0 + var scenarioTimeout = 1800.0 + var keepFixture = false + var scenarioID: String? + var childResult: URL? + var help = false + + init(_ arguments: [String]) throws { + var index = 0 + func value(_ option: String) throws -> String { + index += 1 + guard index < arguments.count else { + throw BenchmarkError.message("missing value for \(option)") + } + return arguments[index] + } + func url(_ value: String) -> URL { URL(fileURLWithPath: value).standardizedFileURL } + while index < arguments.count { + let option = arguments[index] + switch option { + case "--preset": preset = try value(option) + case "--fixture-dir": fixtureDirectory = try url(value(option)) + case "--json": json = try url(value(option)) + case "--compare": compare = try url(value(option)) + case "--filter": filter = try value(option) + case "--iterations": + guard let number = try Int(value(option)), number > 0 else { + throw BenchmarkError.message("iterations must be positive") + } + iterations = number + case "--warmups": + guard let number = try Int(value(option)), number >= 0 else { + throw BenchmarkError.message("warmups must be nonnegative") + } + warmups = number + case "--watch-timeout": + guard let seconds = try Double(value(option)), seconds.isFinite, seconds > 0 else { + throw BenchmarkError.message("watch timeout must be positive") + } + watcherTimeout = seconds + case "--scenario-timeout": + guard let seconds = try Double(value(option)), seconds.isFinite, seconds > 0 else { + throw BenchmarkError.message("scenario timeout must be positive") + } + scenarioTimeout = seconds + case "--keep-fixture": keepFixture = true + case "--scenario": scenarioID = try value(option) + case "--child-result": childResult = try url(value(option)) + case "--help", "-h": help = true + default: throw BenchmarkError.message("unknown option: \(option)") + } + index += 1 + } + _ = try Fixture.count(preset) + } + static let usage = """ + Usage: swift run -c release WorkspaceBenchmarks [options] + --preset small|medium|large Deterministic 1k/10k/100k-file fixture (default small) + --fixture-dir PATH Generate or validate/reuse a preserved fixture + --iterations N Measured iterations per scenario (default 5) + --warmups N Warmups excluded from summaries (default 1; raw samples retained) + --filter TEXT[,TEXT...] Match substrings of backend/persistence/operation + --json PATH Write full report, updated after every scenario + --compare BASELINE.json Print exact I/O deltas and timing ratios + --watch-timeout SECONDS External event deadline (default 15) + --scenario-timeout SECONDS Child process deadline (default 1800) + --keep-fixture Preserve an automatically generated fixture + Every scenario is re-executed in a fresh child. Temporary scenario trees/stores are removed. + """ +} + +@main +struct WorkspaceBenchmarks { + static func main() async { + do { + let options = try Options(Array(CommandLine.arguments.dropFirst())) + if options.help { + print(Options.usage) + return + } + try await run(options) + await TemporaryCleanup.shared.finishInterruptIfNeeded() + } catch { + await TemporaryCleanup.shared.finishInterruptIfNeeded() + FileHandle.standardError.write(Data("WorkspaceBenchmarks: \(error)\n".utf8)) + exit(1) + } + } + static func run(_ options: Options) async throws { + let manager = FileManager.default + let temporary = + (options.childResult?.deletingLastPathComponent() ?? manager.temporaryDirectory) + .appendingPathComponent("WorkspaceBenchmarks-\(UUID().uuidString)") + TemporaryCleanup.shared.register(temporary) + try manager.createDirectory(at: temporary, withIntermediateDirectories: true) + defer { + try? manager.removeItem(at: temporary) + TemporaryCleanup.shared.unregister(temporary) + } + if let scenarioID = options.scenarioID { + guard let directory = options.fixtureDirectory, let resultURL = options.childResult, + let scenario = Scenario.all(preset: options.preset).first(where: { $0.id == scenarioID }) + else { + throw BenchmarkError.message("invalid internal child arguments") + } + // Parent already validated the immutable fixture. Do not regenerate/validate every child. + let fixture = try JSONDecoder().decode( + Fixture.self, from: Data(contentsOf: directory.appendingPathComponent("fixture.json"))) + let runner = ScenarioRunner( + fixture: fixture, sourceTree: directory.appendingPathComponent("tree"), + scenario: scenario, scratch: temporary, watcherTimeout: options.watcherTimeout) + let result = await runner.run(iterations: options.iterations, warmups: options.warmups) + try encoder().encode(result).write(to: resultURL, options: .atomic) + return + } + let directory = + options.fixtureDirectory + ?? manager.temporaryDirectory.appendingPathComponent( + "WorkspaceBenchmarkFixture-\(UUID().uuidString)") + if options.fixtureDirectory == nil, !options.keepFixture { + TemporaryCleanup.shared.register(directory) + } + defer { + if options.fixtureDirectory == nil, !options.keepFixture { + try? manager.removeItem(at: directory) + TemporaryCleanup.shared.unregister(directory) + } + } + if let json = options.json { + try manager.createDirectory( + at: json.deletingLastPathComponent(), withIntermediateDirectories: true) + } + emit("Preparing \(options.preset) fixture at \(directory.path) (excluded from measurements)…") + let fixture = try Fixture.loadOrGenerate(at: directory, preset: options.preset) + let scenarios = Scenario.all(preset: options.preset).filter { scenario in + options.filter.map { filter in + filter.split(separator: ",").contains { scenario.id.contains($0) } + } ?? true + } + guard !scenarios.isEmpty else { throw BenchmarkError.message("filter matched no scenarios") } + var report = Report( + createdAt: Date(), machine: machine(), fixture: fixture, iterations: options.iterations, + warmups: options.warmups, results: [], + measurementNotes: [ + """ + Each scenario uses a child process; fresh prepared stores per sample. Warmups/setup are excluded \ + from timing and counters. Percentiles use linear interpolation. Inner repetitions are averaged; counts are normalized exactly per \ + operation. Independent writes use fresh prepared stores; write1001 always times exactly the 1,001st \ + mutation. + """, + """ + Counters observe backend FileSystem calls, not OS syscalls, draft filesystem, checkpoint-store \ + reads/writes, or glob internals. globMatches reports returned traversal size. + """, + """ + Each sample records getrusage peak immediately before/after timing, normalized to bytes, and its \ + increase. Setup peak is cumulative process high-water, including preceding operations; zero increase \ + does not mean zero allocations. Held RSS is captured with the final result live. Summary uses \ + measured samples only; first-run held RSS is separate. + """, + """ + Store writes are logical estimates from directory metadata before/after, outside timing: append \ + growth or final size of new/replaced files. They exclude transient/deleted files, repeated \ + overwrites and OS write amplification. + """, + """ + Transactions have a baseline checkpoint; transactionFirst has none. Directory incremental capture \ + asserts exactly one changed-file read. Copied/reset regular files are stamped to the old fixture \ + epoch on both OSes. + """, + """ + readCheckpoint is cold index access; readCheckpointWarm primes that index. history1000 cold-loads a \ + fresh directory Workspace; history1000Cached reads the seeded memory Workspace. All mutation \ + scenarios prime preparation with a no-op write. + """, + """ + External event time includes backend write, notification latency/debounce and all recaptures until \ + the changed-path event arrives. Unsupported watchers report errors, not zero-cost successes. + """, + ]) + emit( + """ + \(fixture.fileCount) regular files, \(fixture.bytes) bytes, \(fixture.directoryCount) directories, \ + \(fixture.symlinkCount) symlinks; \(options.warmups) warmup(s) + \(options.iterations) iterations + """ + ) + emit( + """ + scenario median ms p90 ms read MiB reads stat lists \ + entries glob store MiB files setup MiB peak+ MiB held MiB first MiB + """ + ) + for (index, scenario) in scenarios.enumerated() { + try TemporaryCleanup.shared.checkInterrupt() + let resultURL = temporary.appendingPathComponent("result\(index).json") + let child = Process() + guard let executable = Bundle.main.executableURL else { + throw BenchmarkError.message("cannot find benchmark executable") + } + child.executableURL = executable + child.arguments = [ + "--scenario", scenario.id, "--child-result", resultURL.path, + "--fixture-dir", directory.path, "--preset", options.preset, + "--iterations", String(options.iterations), "--warmups", String(options.warmups), + "--watch-timeout", String(options.watcherTimeout), + ] + // Children emit only a JSON file; stderr remains visible for unexpected crashes. + child.standardOutput = FileHandle.nullDevice + try child.run() + TemporaryCleanup.shared.register(child) + let finished = await wait(child, timeout: options.scenarioTimeout) + TemporaryCleanup.shared.unregister(child) + try TemporaryCleanup.shared.checkInterrupt() + let result: ScenarioResult + if finished, child.terminationStatus == 0, manager.fileExists(atPath: resultURL.path) { + do { + result = try JSONDecoder().decode( + ScenarioResult.self, from: Data(contentsOf: resultURL)) + } catch { + result = ScenarioResult( + scenario: scenario, samples: [], peakRSSBytes: 0, + error: "child result failed to decode: \(error)" + ) + } + } else { + result = ScenarioResult( + scenario: scenario, samples: [], peakRSSBytes: 0, + error: finished + ? "child exited \(child.terminationStatus)" + : "child exceeded \(options.scenarioTimeout)s deadline") + } + report.results.append(result) + printRow(result) + if let json = options.json { + try encoder().encode(report).write(to: json, options: .atomic) + } + } + if let baselineURL = options.compare { + let baseline = try JSONDecoder().decode(Report.self, from: Data(contentsOf: baselineURL)) + compare(report, baseline) + } + emit( + """ + Counts exclude persistence-store I/O and backend-internal glob traversal. Raw samples/all call \ + counts are in JSON. * = counts vary; ! = scenario failed. + """ + ) + if options.keepFixture || options.fixtureDirectory != nil { + emit("Fixture preserved: \(directory.path)") + } + if let json = options.json { emit("JSON: \(json.path)") } + if report.results.contains(where: { $0.error != nil }) { + throw BenchmarkError.message( + "one or more scenarios failed; successful results were retained") + } + } + static func encoder() -> JSONEncoder { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return encoder + } + static func wait(_ process: Process, timeout: Double) async -> Bool { + // Waiting on a subprocess must not block a Swift cooperative executor thread. + await withCheckedContinuation { continuation in + DispatchQueue.global().async { + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning, Date() < deadline { Thread.sleep(forTimeInterval: 0.05) } + let finished = !process.isRunning + if !finished { + process.terminate() + Thread.sleep(forTimeInterval: 0.5) + if process.isRunning { kill(process.processIdentifier, SIGKILL) } + } + // Do not wait a second time after Foundation reports child exit. + continuation.resume(returning: finished) + } + } + } + static func machine() -> [String: String] { + var name = utsname() + uname(&name) + let machineSize = MemoryLayout.size(ofValue: name.machine) + let architecture = withUnsafePointer(to: &name.machine) { + $0.withMemoryRebound(to: CChar.self, capacity: machineSize) { String(cString: $0) } + } + func command(_ executable: String, _ arguments: [String]) -> String { + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + let pipe = Pipe() + process.standardOutput = pipe + process.standardError = pipe + do { + try process.run() + let data = pipe.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + return String(decoding: data, as: UTF8.self).trimmingCharacters( + in: .whitespacesAndNewlines) + } catch { return "unavailable" } + } + #if DEBUG + let configuration = "debug" + #else + let configuration = "release" + #endif + return [ + "buildConfiguration": configuration, + "os": ProcessInfo.processInfo.operatingSystemVersionString, "architecture": architecture, + "logicalCPUs": String(ProcessInfo.processInfo.processorCount), + "physicalMemoryBytes": String(ProcessInfo.processInfo.physicalMemory), + "swift": command("/usr/bin/env", ["swift", "--version"]), + "gitCommit": command("/usr/bin/env", ["git", "rev-parse", "HEAD"]), + "gitBranch": command("/usr/bin/env", ["git", "branch", "--show-current"]), + "gitStatus": command("/usr/bin/env", ["git", "status", "--short"]), + ] + } + static func printRow(_ result: ScenarioResult) { + let id = result.scenario.id.padding(toLength: 41, withPad: " ", startingAt: 0) + guard let median = result.medianMS, let p90 = result.p90MS, let io = result.samples.first?.io + else { + emit("\(id) ! \(result.error ?? "no samples")") + return + } + func mib(_ bytes: Int64?) -> String { + bytes.map { String(format: "%10.2f", Double($0) / 1_048_576) } ?? " -" + } + let sample = result.samples.first + let setup = result.measuredOperations.map(\.setupPeakRSSBytes).max() + let increase = result.measuredOperations.map(\.operationPeakIncreaseBytes).max() + emit( + id + + String( + format: " %10.3f %9.3f %10.2f %8lld %7lld %7lld %8lld %8lld %10.3f %6lld", + median, p90, Double(io.bytesRead) / 1_048_576, io.filesRead, + io.calls["stat", default: 0], io.calls["listDirectory", default: 0], + io.directoryEntries, io.globMatches, + Double(sample?.storeBytesWritten ?? 0) / 1_048_576, sample?.storeFilesWritten ?? 0 + ) + mib(setup) + mib(increase) + mib(result.heldRSSDeltaBytes) + + mib(result.firstRunHeldRSSDeltaBytes) + (result.deterministicIO ? "" : " *")) + if let error = result.error { emit(" ! Partial samples: \(error)") } + } + static func compare(_ current: Report, _ baseline: Report) { + emit( + "\nComparison with \(baseline.fixture.preset) baseline (current/baseline time; exact current−baseline counts):" + ) + if current.fixture.preset != baseline.fixture.preset + || current.fixture.version != baseline.fixture.version + { + emit("Fixture differs; this comparison measures scaling, not a performance regression.") + } + for result in current.results { + guard + let previous = baseline.results.first(where: { $0.scenario.id == result.scenario.id }), + let median = result.medianMS, let oldMedian = previous.medianMS, + let p90 = result.p90MS, let oldP90 = previous.p90MS, + let io = result.samples.first?.io, let oldIO = previous.samples.first?.io + else { continue } + let calls = Set(io.calls.keys).union(oldIO.calls.keys).sorted().map { + "\($0)=\(io.calls[$0, default: 0] - oldIO.calls[$0, default: 0])" + }.joined(separator: " ") + emit( + String( + format: "%@ median %.2fx p90 %.2fx", result.scenario.id, median / oldMedian, + p90 / oldP90) + + " bytes=\(io.bytesRead - oldIO.bytesRead) reads=\(io.filesRead - oldIO.filesRead)" + + " entries=\(io.directoryEntries - oldIO.directoryEntries)" + + " globMatches=\(io.globMatches - oldIO.globMatches) \(calls)" + + " storeBytes=\((result.samples.first?.storeBytesWritten ?? 0) - (previous.samples.first?.storeBytesWritten ?? 0))" + + " storeFiles=\((result.samples.first?.storeFilesWritten ?? 0) - (previous.samples.first?.storeFilesWritten ?? 0))" + + (result.deterministicIO && previous.deterministicIO + ? "" : " [counts vary; delta uses first sample]")) + } + } +} + +#endif + +#if !os(macOS) && !os(Linux) +@main +struct WorkspaceBenchmarks { + static func main() { + print("WorkspaceBenchmarks requires macOS or Linux.") + } +} +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift new file mode 100644 index 0000000..c8fdf6c --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift @@ -0,0 +1,153 @@ +#if os(macOS) || os(Linux) +import Foundation +import Workspace + +/// Counts calls at the decorator boundary, not syscalls performed within the backend. +/// In particular glob/copy/remove and checkpoint-store disk I/O are opaque here. +struct IOCounts: Codable, Equatable, Sendable { + var calls: [String: Int64] = [:] + var bytesRead: Int64 = 0 + var filesRead: Int64 = 0 // successful read invocations, including range/stream reads + var directoryEntries: Int64 = 0 + var globMatches: Int64 = 0 +} + +/// A lock only protects counter updates; no lock is held across an await. +class CountingFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { + let base: any FileSystem + private let lock = NSLock() + private var counts = IOCounts() + init(_ base: any FileSystem) { self.base = base } + func reset() { lock.withLock { counts = IOCounts() } } + func snapshot() -> IOCounts { lock.withLock { counts } } + func call(_ name: String) { lock.withLock { counts.calls[name, default: 0] += 1 } } + private func read(_ data: Data, file: Bool = true) { + lock.withLock { + counts.bytesRead += Int64(data.count) + if file { counts.filesRead += 1 } + } + } + func stat(path: WorkspacePath) async throws -> FileInfo { + call("stat") + return try await base.stat(path: path) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + call("listDirectory") + let entries = try await base.listDirectory(path: path) + lock.withLock { counts.directoryEntries += Int64(entries.count) } + return entries + } + func readFile(path: WorkspacePath) async throws -> Data { + call("readFile") + let data = try await base.readFile(path: path) + read(data) + return data + } + func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + call("readRange") + let data = try await base.readFile(path: path, offset: offset, length: length) + read(data) + return data + } + func readFileChunks(path: WorkspacePath, chunkSize: Int) async throws -> AsyncThrowingStream< + Data, Error + > { + call("readChunks") + let stream = try await base.readFileChunks(path: path, chunkSize: chunkSize) + lock.withLock { counts.filesRead += 1 } + return AsyncThrowingStream { continuation in + let task = Task { + do { + for try await data in stream { + self.read(data, file: false) + continuation.yield(data) + } + continuation.finish() + } catch { continuation.finish(throwing: error) } + } + continuation.onTermination = { _ in task.cancel() } + } + } + func exists(path: WorkspacePath) async -> Bool { + call("exists") + return await base.exists(path: path) + } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + call("glob") + let paths = try await base.glob(pattern: pattern, currentDirectory: currentDirectory) + lock.withLock { counts.globMatches += Int64(paths.count) } + return paths + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + call("writeFile") + try await base.writeFile(path: path, data: data, append: append) + } + func createFile(path: WorkspacePath, data: Data) async throws { + call("createFile") + try await base.createFile(path: path, data: data) + } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { + call("createDirectory") + try await base.createDirectory(path: path, recursive: recursive) + } + func remove(path: WorkspacePath, recursive: Bool) async throws { + call("remove") + try await base.remove(path: path, recursive: recursive) + } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { + call("move") + try await base.move(from: sourcePath, to: destinationPath) + } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) + async throws + { + call("copy") + try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) + } + func createSymlink(path: WorkspacePath, target: String) async throws { + call("createSymlink") + try await base.createSymlink(path: path, target: target) + } + func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { + call("createHardLink") + try await base.createHardLink(path: path, target: target) + } + func readSymlink(path: WorkspacePath) async throws -> String { + call("readSymlink") + return try await base.readSymlink(path: path) + } + func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { + call("setPermissions") + try await base.setPermissions(path: path, permissions: permissions) + } + func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { + call("resolveRealPath") + return try await base.resolveRealPath(path: path) + } + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + call("resolvePotentialPath") + guard let resolver = base as? any PotentialPathResolvingFileSystem else { + throw WorkspaceError.unsupported("benchmark backend must support potential paths") + } + return try await resolver.resolvePotentialPath(path: path) + } + func capabilities() async -> FileSystemFeatures { + call("capabilities") + return await base.capabilities() + } +} + +/// Only Local advertises observation; all filesystem forwarding is inherited. +final class ObservedCountingFileSystem: CountingFileSystem, FileSystemChangeSource, + @unchecked Sendable +{ + func changes() async throws -> AsyncStream { + call("changes") + guard let source = base as? any FileSystemChangeSource else { + throw BenchmarkError.message("backend does not provide filesystem notifications") + } + return try await source.changes() + } +} + +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/Fixture.swift b/Benchmarks/WorkspaceBenchmarks/Fixture.swift new file mode 100644 index 0000000..dbda5cc --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/Fixture.swift @@ -0,0 +1,191 @@ +#if os(macOS) || os(Linux) +import Foundation +import Workspace +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif + +struct Fixture: Codable, Sendable { + static let modificationDate = Date(timeIntervalSince1970: 1_600_000_000) + static let version = 1 + var version: Int = Self.version + var preset: String + var fileCount: Int + var bytes: Int64 + var symlinkCount: Int + var directoryCount: Int + + static func count(_ preset: String) throws -> Int { + switch preset { + case "small": 1_000 + case "medium": 10_000 + case "large": 100_000 + default: throw BenchmarkError.message("unknown preset: \(preset)") + } + } + static func path(_ i: Int, preset: String) -> String { + if preset == "large" { + return + "node_modules/group\(i / 1000)/package\(i / 100)/node_modules/dependency/src/internal/generated/file\(i).swift" + } + return "Sources/module\(i / 100)/feature\(i / 20)/file\(i).\(i % 100 == 99 ? "bin" : "swift")" + } + static func data(_ i: Int, preset: String) -> Data { + let size = + preset == "large" + ? (i < 50 ? 4_096 : 1_024) : (i % 100 == 99 ? 819_200 : (i % 100 >= 90 ? 102_400 : 4_096)) + if i % 100 == 99 { + return Data((0.. Fixture { + let manager = FileManager.default + let manifest = directory.appendingPathComponent("fixture.json") + if manager.fileExists(atPath: manifest.path) { + let fixture = try JSONDecoder().decode(Fixture.self, from: Data(contentsOf: manifest)) + guard fixture.version == version, fixture.preset == preset, + fixture.fileCount == (try count(preset)), + fixture.symlinkCount == max(5, fixture.fileCount / 200), + (try manager.attributesOfItem(atPath: directory.appendingPathComponent("tree").path))[ + .type] as? FileAttributeType == .typeDirectory + else { + throw BenchmarkError.message( + "fixture version/preset mismatch or missing tree: \(directory.path)") + } + // Validate contents and the complete path set on reuse, outside measurement. + // Never overwrite a user's tree or silently benchmark additional files. + var expected: Set = ["links"] + for i in 0.. String? in + (item as? URL).map { String($0.path.dropFirst(tree.path.count + 1)) } + }) + guard actual == expected, + fixture.directoryCount == expected.count - fixture.fileCount - fixture.symlinkCount + 1 + else { + throw BenchmarkError.message("fixture tree contains missing or unexpected entries") + } + var validatedBytes: Int64 = 0 + for i in 0.. = ["", "links"] + let count = try count(preset) + var bytes: Int64 = 0 + for i in 0.. InMemoryFileSystem { + let local = try LocalFileSystem(root: tree) + let memory = InMemoryFileSystem() + func copy(_ path: WorkspacePath) async throws { + for entry in try await local.listDirectory(path: path) { + try TemporaryCleanup.shared.checkInterrupt() + let path = entry.info.path + switch entry.info.kind { + case .directory: + try await memory.createDirectory(path: path, recursive: true) + try await copy(path) + case .file: + try await memory.writeFile(path: path, data: local.readFile(path: path), append: false) + case .symlink: + try await memory.createSymlink(path: path, target: local.readSymlink(path: path)) + } + } + } + try await copy(.root) + return memory + } +} + +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/Measurement.swift b/Benchmarks/WorkspaceBenchmarks/Measurement.swift new file mode 100644 index 0000000..7c1c038 --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/Measurement.swift @@ -0,0 +1,124 @@ +#if os(macOS) || os(Linux) +import Foundation +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif + +struct Sample: Codable, Sendable { + var milliseconds: Double + var io: IOCounts + var repetitions: Int = 1 + var heldRSSDeltaBytes: Int64? + var rssBeforeBytes: Int64? + var rssHeldBytes: Int64? + var setupPeakRSSBytes: Int64 + var operationPeakRSSBytes: Int64 + var operationPeakIncreaseBytes: Int64 + var storeBytesWritten: Int64 = 0 + var storeFilesWritten: Int64 = 0 + var innerSamples: [Sample] = [] +} + +struct Scenario: Codable, Sendable { + var operation: Operation + var backend: Backend + var persistence: Persistence + var id: String { "\(backend.rawValue)/\(persistence.rawValue)/\(operation.rawValue)" } + + static func all(preset: String) -> [Scenario] { + Backend.allCases.flatMap { backend in + Operation.allCases.flatMap { operation -> [Scenario] in + guard !(preset == "large" && backend == .memory), + !(operation.localOnly && backend == .memory) + else { return [] } + return operation.persistences.compactMap { persistence in + guard !(backend == .memory && persistence == .directory) else { return nil } + return Scenario(operation: operation, backend: backend, persistence: persistence) + } + } + } + } +} +struct ScenarioResult: Codable { + var scenario: Scenario + var samples: [Sample] + var warmupSamples: [Sample] = [] + var peakRSSBytes: Int64 + var error: String? + var medianMS: Double? { percentile(0.5) } + var p90MS: Double? { percentile(0.9) } + // Linear interpolation between ranked samples; retain all raw measurements. + func percentile(_ fraction: Double) -> Double? { + let sorted = samples.map(\.milliseconds).sorted() + guard !sorted.isEmpty else { return nil } + let rank = Double(sorted.count - 1) * fraction + let lower = Int(floor(rank)) + let upper = Int(ceil(rank)) + return sorted[lower] + (sorted[upper] - sorted[lower]) * (rank - Double(lower)) + } + var measuredOperations: [Sample] { + samples.flatMap { $0.innerSamples.isEmpty ? [$0] : $0.innerSamples } + } + var heldRSSDeltaBytes: Int64? { measuredOperations.compactMap(\.heldRSSDeltaBytes).max() } + var firstRunHeldRSSDeltaBytes: Int64? { + (warmupSamples.first ?? samples.first)?.heldRSSDeltaBytes + } + var deterministicIO: Bool { samples.dropFirst().allSatisfy { $0.io == samples.first?.io } } +} +struct Report: Codable { + var schemaVersion = 2 + var createdAt: Date + var machine: [String: String] + var fixture: Fixture + var iterations: Int + var warmups: Int + var results: [ScenarioResult] + var measurementNotes: [String] +} + +enum BenchmarkError: Error, CustomStringConvertible { + case message(String) + var description: String { + switch self { + case .message(let message): message + } + } +} + +func peakRSS() -> Int64 { + var usage = rusage() + #if canImport(Darwin) + getrusage(RUSAGE_SELF, &usage) + return Int64(usage.ru_maxrss) + #else + getrusage(Int32(RUSAGE_SELF.rawValue), &usage) + return Int64(usage.ru_maxrss) * 1024 + #endif +} +func currentRSS() -> Int64? { + #if canImport(Darwin) + var info = mach_task_basic_info() + var count = mach_msg_type_number_t( + MemoryLayout.size / MemoryLayout.size) + let status = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), $0, &count) + } + } + return status == KERN_SUCCESS ? Int64(info.resident_size) : nil + #else + guard let contents = try? String(contentsOfFile: "/proc/self/statm", encoding: .utf8), + let pages = contents.split(separator: " ").dropFirst().first.flatMap({ Int64($0) }) + else { return nil } + return pages * Int64(sysconf(Int32(_SC_PAGESIZE))) + #endif +} +func milliseconds(since start: ContinuousClock.Instant) -> Double { + let components = start.duration(to: ContinuousClock.now).components + return Double(components.seconds) * 1_000 + Double(components.attoseconds) / 1e15 +} +func emit(_ message: String) { FileHandle.standardOutput.write(Data((message + "\n").utf8)) } + +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/Operations.swift b/Benchmarks/WorkspaceBenchmarks/Operations.swift new file mode 100644 index 0000000..4c489ae --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/Operations.swift @@ -0,0 +1,103 @@ +#if os(macOS) || os(Linux) +import Foundation + +enum Backend: String, CaseIterable, Codable, Sendable { + case local, memory +} + +enum Persistence: String, CaseIterable, Codable, Sendable { + case memory, directory +} + +enum Operation: String, CaseIterable, Codable, Sendable { + case writeText, apply50, preview, transaction, transactionFirst + case checkpointFirst, checkpointIncremental, diff, restore + case readCheckpoint, readCheckpointWarm, readCurrent, search, glob + case history1000, history1000Cached, write1001, limitedWrite, eventsInitial, eventsExternal + + var localOnly: Bool { + switch self { + case .history1000, .write1001, .limitedWrite, .eventsInitial, .eventsExternal: true + case .writeText, .apply50, .preview, .transaction, .transactionFirst, + .checkpointFirst, .checkpointIncremental, .diff, .restore, + .readCheckpoint, .readCheckpointWarm, .readCurrent, .search, .glob, .history1000Cached: + false + } + } + + var persistences: [Persistence] { + switch self { + case .writeText, .apply50, .transaction, .transactionFirst, .checkpointFirst, + .checkpointIncremental, .diff, .restore, .readCheckpoint, .readCheckpointWarm: + [.memory, .directory] + case .history1000, .write1001: [.directory] + case .preview, .readCurrent, .search, .glob, .history1000Cached, + .limitedWrite, .eventsInitial, .eventsExternal: + [.memory] + } + } + + var needsBaselineCheckpoint: Bool { + switch self { + case .transaction, .checkpointIncremental, .diff, .restore, .readCheckpoint, + .readCheckpointWarm: + true + case .writeText, .apply50, .preview, .transactionFirst, .checkpointFirst, + .readCurrent, .search, .glob, .history1000, .history1000Cached, + .write1001, .limitedWrite, .eventsInitial, .eventsExternal: + false + } + } + + var needsChangedFile: Bool { + switch self { + case .checkpointIncremental, .diff, .restore: true + case .writeText, .apply50, .preview, .transaction, .transactionFirst, .checkpointFirst, + .readCheckpoint, .readCheckpointWarm, .readCurrent, .search, .glob, + .history1000, .history1000Cached, .write1001, .limitedWrite, .eventsInitial, + .eventsExternal: + false + } + } + + var needsHistory: Bool { + switch self { + case .history1000, .history1000Cached, .write1001: true + case .writeText, .apply50, .preview, .transaction, .transactionFirst, .checkpointFirst, + .checkpointIncremental, .diff, .restore, .readCheckpoint, .readCheckpointWarm, + .readCurrent, .search, .glob, .limitedWrite, .eventsInitial, .eventsExternal: + false + } + } + + var observes: Bool { + switch self { + case .eventsInitial, .eventsExternal: true + case .writeText, .apply50, .preview, .transaction, .transactionFirst, .checkpointFirst, + .checkpointIncremental, .diff, .restore, .readCheckpoint, .readCheckpointWarm, + .readCurrent, .search, .glob, .history1000, .history1000Cached, .write1001, .limitedWrite: + false + } + } + + var resetFileCount: Int { self == .apply50 ? 50 : 1 } + + // Read-only repetitions share the primed Workspace. Mutations get independent + // prepared Workspace/store instances, so each write1001 really is the 1,001st. + var repetitions: Int { + switch self { + case .readCurrent, .history1000Cached: 100 + case .readCheckpointWarm: 20 + case .writeText: 20 + case .write1001: 5 + case .apply50, .preview, .transaction, .transactionFirst, .checkpointFirst, + .checkpointIncremental, .diff, .restore, .readCheckpoint, .search, + .glob, .history1000, .limitedWrite, .eventsInitial, .eventsExternal: + 1 + } + } + + var independentRepetitions: Bool { self == .writeText || self == .write1001 } +} + +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/Scenarios.swift b/Benchmarks/WorkspaceBenchmarks/Scenarios.swift new file mode 100644 index 0000000..71926ce --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/Scenarios.swift @@ -0,0 +1,342 @@ +#if os(macOS) || os(Linux) +import Foundation +import Workspace + +struct ScenarioRunner { + let fixture: Fixture + let sourceTree: URL + let scenario: Scenario + let scratch: URL + let watcherTimeout: Double + + func run(iterations: Int, warmups: Int) async -> ScenarioResult { + var result = ScenarioResult(scenario: scenario, samples: [], peakRSSBytes: 0) + do { + let tree = scratch.appendingPathComponent("tree") + let base: any FileSystem + switch scenario.backend { + case .local: + try FileManager.default.copyItem(at: sourceTree, to: tree) + // Foundation's Linux copy does not preserve mtimes. Stamp every + // regular file, never symlinks, before a baseline can be captured. + for i in 0.. Sample { + guard var result = samples.first, samples.allSatisfy({ $0.io == result.io }) else { + throw BenchmarkError.message("independent repetition counts differ") + } + result.milliseconds = samples.map(\.milliseconds).reduce(0, +) / Double(samples.count) + result.repetitions = samples.count + result.innerSamples = samples + // Keep the first operation's exact memory/store pair as the representative. + // Summary maxima come from innerSamples, never mismatched before/after fields. + return result + } + + private func measure( + base: any FileSystem, counter: CountingFileSystem, tree: URL, storeName: String + ) async throws -> Sample { + let operation = scenario.operation + let path = try WorkspacePath(Fixture.path(0, preset: fixture.preset)) + let original = Fixture.data(0, preset: fixture.preset) + let originalText = String(decoding: original, as: UTF8.self) + var changed = original + changed[changed.startIndex + 3] = UInt8(ascii: "X") + let changedText = String(decoding: changed, as: UTF8.self) + let edits = try (0..<50).map { i -> Edit in + var data = Fixture.data(i, preset: fixture.preset) + data[data.startIndex + 3] = UInt8(ascii: "X") + return .writeText( + try WorkspacePath(Fixture.path(i, preset: fixture.preset)), + String(decoding: data, as: UTF8.self) + ) + } + for i in 0..? + if operation == .eventsExternal { + externalStream = await workspace.events(.init(includeCheckpoints: false)) + try assertObservation(counter) + try await Task.sleep(for: .milliseconds(300)) + } + let storeBefore = try StoreFootprint.capture(store) + counter.reset() + let rssBefore = currentRSS() + let setupPeak = peakRSS() + var rssHeld: Int64? + var heldResult: Any? + let repetitions = operation.independentRepetitions ? 1 : operation.repetitions + let start = ContinuousClock.now + for _ in 0..= fixture.fileCount else { + throw BenchmarkError.message("glob missed files") + } + heldResult = paths + case .history1000, .history1000Cached: + let mutations = try await historyReader.history() + guard mutations.count == 1000 else { + throw BenchmarkError.message("expected 1000 mutations, got \(mutations.count)") + } + heldResult = mutations + case .eventsInitial: + let stream = await workspace.events(.init(includeCheckpoints: false)) + externalStream = stream + heldResult = stream + try assertObservation(counter) + case .eventsExternal: + guard let stream = externalStream else { + throw BenchmarkError.message("missing external stream") + } + try await base.writeFile(path: path, data: changed, append: false) + try await waitForChange(stream, path: path) + heldResult = stream + } + } + let elapsed = milliseconds(since: start) / Double(repetitions) + let operationPeak = peakRSS() + withExtendedLifetime(heldResult) { + if rssHeld == nil { rssHeld = currentRSS() } + } + let counts = try counter.snapshot().perOperation(repetitions) + if operation == .checkpointIncremental, scenario.persistence == .directory, + counts.filesRead != 1 || counts.bytesRead != Int64(changed.count) + { + throw BenchmarkError.message( + "incremental checkpoint premise failed: \(counts.filesRead) reads") + } + let storeWritten = try StoreFootprint.capture(store).written(since: storeBefore) + if let stream = externalStream { await cancel(stream) } + switch operation { + case .writeText, .apply50, .transaction, .transactionFirst, .limitedWrite, .write1001: + guard try await base.readFile(path: path) == changed else { + throw BenchmarkError.message("mutation result mismatch") + } + case .restore, .preview: + guard try await base.readFile(path: path) == original else { + throw BenchmarkError.message("restoration/preview mismatch") + } + case .checkpointFirst, .checkpointIncremental, .diff, .readCheckpoint, .readCheckpointWarm, + .readCurrent, .search, .glob, .history1000, .history1000Cached, .eventsInitial, + .eventsExternal: + break + } + return Sample( + milliseconds: elapsed, io: counts, repetitions: repetitions, + heldRSSDeltaBytes: delta(rssBefore, rssHeld), rssBeforeBytes: rssBefore, + rssHeldBytes: rssHeld, + setupPeakRSSBytes: setupPeak, operationPeakRSSBytes: operationPeak, + operationPeakIncreaseBytes: max(0, operationPeak - setupPeak), + storeBytesWritten: storeWritten.bytes, storeFilesWritten: storeWritten.files + ) + } + + private func assertObservation(_ counter: CountingFileSystem) throws { + guard counter.snapshot().filesRead >= Int64(fixture.fileCount) else { + throw BenchmarkError.message( + "notification setup unavailable: events() did not capture the initial tree") + } + } + + private func delta(_ before: Int64?, _ after: Int64?) -> Int64? { + guard let before, let after else { return nil } + return after - before + } + + private func cancel(_ stream: AsyncStream) async { + let consumer = Task { + for await _ in stream { if Task.isCancelled { break } } + } + consumer.cancel() + await consumer.value + await Task.yield() + } + + private func waitForChange(_ stream: AsyncStream, path: WorkspacePath) + async throws + { + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + for await event in stream { + if case .changes(let changes) = event, changes.touchedPaths.contains(path) { return } + } + throw BenchmarkError.message("notification stream ended before external change") + } + group.addTask { + try await Task.sleep(for: .seconds(watcherTimeout)) + throw BenchmarkError.message("external change event timed out after \(watcherTimeout)s") + } + defer { group.cancelAll() } + _ = try await group.next() + } + } +} + +extension IOCounts { + func perOperation(_ repetitions: Int) throws -> IOCounts { + let divisor = Int64(repetitions) + let values = Array(calls.values) + [bytesRead, filesRead, directoryEntries, globMatches] + guard values.allSatisfy({ $0.isMultiple(of: divisor) }) else { + throw BenchmarkError.message("repetition counts cannot be normalized exactly") + } + return IOCounts( + calls: calls.mapValues { $0 / divisor }, bytesRead: bytesRead / divisor, + filesRead: filesRead / divisor, directoryEntries: directoryEntries / divisor, + globMatches: globMatches / divisor + ) + } +} + +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/StoreFootprint.swift b/Benchmarks/WorkspaceBenchmarks/StoreFootprint.swift new file mode 100644 index 0000000..84d6e90 --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/StoreFootprint.swift @@ -0,0 +1,58 @@ +#if os(macOS) || os(Linux) +import Foundation + +/// Logical persistence writes estimated outside timing. Appends count growth; +/// replaced/new files count their final size. This cannot see overwritten bytes +/// beyond the final extent, deleted/transient files, or filesystem write amplification. +struct StoreFootprint { + struct File { + var size: Int64 + var modificationDate: Date? + var inode: UInt64? + } + var files: [String: File] = [:] + + static func capture(_ directory: URL) throws -> StoreFootprint { + let manager = FileManager.default + guard manager.fileExists(atPath: directory.path) else { return StoreFootprint() } + guard let enumerator = manager.enumerator(atPath: directory.path) else { + throw BenchmarkError.message("cannot enumerate checkpoint store") + } + var result = StoreFootprint() + for case let path as String in enumerator { + let attributes = try manager.attributesOfItem( + atPath: directory.appendingPathComponent(path).path) + guard attributes[.type] as? FileAttributeType == .typeRegular else { continue } + result.files[path] = File( + size: (attributes[.size] as? NSNumber)?.int64Value ?? 0, + modificationDate: attributes[.modificationDate] as? Date, + inode: (attributes[.systemFileNumber] as? NSNumber)?.uint64Value + ) + } + return result + } + + func written(since before: StoreFootprint) -> (bytes: Int64, files: Int64) { + var bytes: Int64 = 0 + var count: Int64 = 0 + for (path, file) in files { + if let old = before.files[path] { + // Acquiring a store lock can touch an unchanged empty lock file. + // This is metadata activity, not a content write. + if old.size == 0, file.size == 0, old.inode == file.inode { continue } + guard + old.size != file.size || old.modificationDate != file.modificationDate + || old.inode != file.inode + else { continue } + bytes += + old.inode == file.inode && file.size > old.size ? file.size - old.size : file.size + } else { + bytes += file.size + } + count += 1 + } + return (bytes, count) + } +} + +#endif diff --git a/Benchmarks/WorkspaceBenchmarks/TemporaryCleanup.swift b/Benchmarks/WorkspaceBenchmarks/TemporaryCleanup.swift new file mode 100644 index 0000000..9cf6319 --- /dev/null +++ b/Benchmarks/WorkspaceBenchmarks/TemporaryCleanup.swift @@ -0,0 +1,82 @@ +#if os(macOS) || os(Linux) +import Foundation +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif + +/// Signal handlers run on a dispatch queue, never in an async-signal context. +/// The parent's temporary root contains child scratch trees as a fallback. +final class TemporaryCleanup: @unchecked Sendable { + static let shared = TemporaryCleanup() + private let lock = NSLock() + private var roots: Set = [] + private var children: [Process] = [] + private var mainStopped = false + private var interruptSignal: Int32? + private var sources: [DispatchSourceSignal] = [] + + private init() { + for number in [SIGINT, SIGTERM] { + signal(number, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: number, queue: .global()) + source.setEventHandler { [weak self] in + guard let self else { return } + let state = self.lock.withLock { + self.interruptSignal = number + return (self.roots, self.children) + } + for child in state.1 where child.isRunning { kill(child.processIdentifier, number) } + let deadline = Date().addingTimeInterval(1) + while state.1.contains(where: { $0.isRunning }), Date() < deadline { + Thread.sleep(forTimeInterval: 0.01) + } + for child in state.1 where child.isRunning { kill(child.processIdentifier, SIGKILL) } + // Wait until the main path stops creating files. Synchronous fixture + // generation checks the flag between files; async runs check between + // operations. The parent forcibly stops slow children before cleanup. + while !self.lock.withLock({ self.mainStopped }) { + Thread.sleep(forTimeInterval: 0.01) + } + let roots = self.lock.withLock { self.roots } + for root in roots { try? FileManager.default.removeItem(at: root) } + exit(128 + number) + } + source.resume() + sources.append(source) + } + } + + // Keep main alive while the handler interrupts children and removes roots. + // Otherwise a quickly exiting child can make main exit before cleanup finishes. + func finishInterruptIfNeeded() async { + guard + lock.withLock({ + guard interruptSignal != nil else { return false } + mainStopped = true + return true + }) + else { return } + while true { try? await Task.sleep(for: .milliseconds(10)) } + } + + func checkInterrupt() throws { + if lock.withLock({ interruptSignal != nil }) { + throw BenchmarkError.message("interrupted") + } + } + + func register(_ root: URL) { lock.withLock { _ = roots.insert(root) } } + func unregister(_ root: URL) { lock.withLock { _ = roots.remove(root) } } + func register(_ child: Process) { + let interrupted = lock.withLock { + children.append(child) + return interruptSignal != nil + } + if interrupted { kill(child.processIdentifier, SIGKILL) } + } + func unregister(_ child: Process) { lock.withLock { children.removeAll { $0 === child } } } +} + +#endif diff --git a/Package.swift b/Package.swift index 9dc40ea..51d4525 100644 --- a/Package.swift +++ b/Package.swift @@ -22,6 +22,11 @@ let package = Package( dependencies: [.target(name: "WorkspacePOSIX", condition: .when(platforms: [.linux]))] ), .target(name: "WorkspacePOSIX"), + .executableTarget( + name: "WorkspaceBenchmarks", + dependencies: ["Workspace"], + path: "Benchmarks/WorkspaceBenchmarks" + ), .testTarget( name: "WorkspaceTests", dependencies: ["Workspace"] diff --git a/README.md b/README.md index 39b5457..ab43b59 100644 --- a/README.md +++ b/README.md @@ -423,3 +423,56 @@ swift test --disable-xctest --enable-swift-testing ``` CI runs Swift 6.2 tests on macOS and Linux. + +## Benchmarks + +The separate executable does not run in `swift test`. Run in release mode: + +```sh +swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/small.json +swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/after.json \ + --compare Benchmarks/Results/small.json +swift run -c release WorkspaceBenchmarks --preset medium --filter transaction,limitedWrite +``` + +Defaults: one warmup, five measured samples, a fresh child per scenario. Fast operations +use inner repetitions with per-operation times/counts; writes use independent prepared +stores (each `write1001` is exactly the 1,001st mutation). Fixtures have 1k/10k/100k +regular files: small ~20 MiB, medium ~200 MiB, large ~98 MiB with deep dependency nesting. +Allow roughly **2–5 minutes for small**, **10–30 minutes for medium**, and **well over +an hour for large** at defaults, depending on filesystem and watcher availability. +Fixture generation is excluded. `--fixture-dir PATH` generates/validates/reuses a +preserved fixture; otherwise temporary trees/stores are removed on completion or SIGINT. +`Benchmarks/Results/` is ignored. Use `--help` for filters, iteration counts and +deadlines (30 minutes per child by default). + +| Scenario | What's timed | Setup excluded | Counter blind spots | +| --- | --- | --- | --- | +| `writeText`, `apply50`, `limitedWrite` | One recorded write, 50 edits, quota-limited write | No-op preparation, file reset | Checkpoint store; Limited metadata traversal is visible | +| `transaction` / `transactionFirst` | Begin, one draft write, commit | Preparation; baseline checkpoint for `transaction` only | Draft filesystem, checkpoint store | +| `preview` | One edit preview | Preparation | Draft filesystem | +| `checkpointFirst` / `checkpointIncremental` | First capture / capture after one change | Preparation; baseline and change for incremental | Checkpoint store | +| `diff`, `restore` | Checkpoint-to-current diff / restore after one change | Preparation, baseline, change | Checkpoint store | +| `readCheckpoint` / `readCheckpointWarm`, `readCurrent` | Cold / primed checkpoint index read; direct current read | Baseline; first read for warm variant | Checkpoint store | +| `search`, `glob` | Full absent-literal search / `**/*` glob | Preparation | Glob internals; returned `globMatches` is visible | +| `history1000` / `history1000Cached` | Fresh directory Workspace log load/decode / cached memory history | Seed 1,000 mutations; new reader for cold variant | Checkpoint store | +| `write1001` | Exactly the 1,001st recorded directory write | Preparation and 1,000 mutations | Checkpoint store | +| `eventsInitial`, `eventsExternal` | Enable observation / direct external write until its event arrives | Preparation; initial observation and drain for external | External write bypasses counter; recaptures are visible | + +The table shows median/p90 wall time, backend read bytes/calls, stat/list calls, entries, +glob matches, logical store bytes/files written, setup RSS high-water, operation increase +in that high-water, and held RSS delta. Held RSS is sampled with the result live. The +first-run column is the live-footprint figure: after the first run, allocator reuse hides +most of a result's memory, so the measured-sample summary (which excludes warmups) can be +near zero or slightly negative. Zero high-water growth can likewise mean allocator reuse. +Percentiles use linear interpolation between samples. Setup high-water includes preceding +samples in the same child; JSON retains each sample's values and process peak. +Darwin/Linux RSS units are normalized to bytes. Store write counts include newly created +empty lock files. + +Counters observe the `FileSystem` boundary, not syscalls. Store writes are estimated +outside timing from directory metadata: append growth or final size of new/replaced +files; transient/deleted files, repeated overwrites and OS write amplification are +invisible. Compare identical presets for regressions: timing ratios and exact backend +count deltas are printed. Watcher errors produce a nonzero exit while retaining successful +results; run outside a restrictive macOS sandbox to measure FSEvents. diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index ae4d060..ab69f85 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -216,7 +216,7 @@ public protocol FileSystemChangeSource: FileSystem { /// Resolves links in the backend's namespace without requiring the target to exist. /// Missing parent suffixes are retained for quota preflight without creating them. -protocol PotentialPathResolvingFileSystem: FileSystem { +package protocol PotentialPathResolvingFileSystem: FileSystem { func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath } diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index 1a7b019..af9da7e 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -511,7 +511,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { return current } - func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0, allowMissing: true) } diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index b1f6ef1..b465de5 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -75,7 +75,7 @@ public actor LimitedFileSystem: FileSystem, PotentialPathResolvingFileSystem { public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { try await base.resolveRealPath(path: path) } - func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { try await pathResolver.resolve(path, features: await base.capabilities(), followFinal: true) } public func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index ad6e008..f2c1514 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -898,7 +898,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem return entry.path } - func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { try resolve(path, followFinal: true, allowMissingAncestors: true).path } diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index eb4964b..9246a09 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -282,7 +282,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst // Used only when a symlink target or suffix is missing. Resolve inside the selected // backend before translating back; raw absolute targets are never outer virtual paths. - func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { let resolved = resolveWritable(path: path, mounts: mountsSnapshot()) let real = try await FileSystemPathResolver(base: resolved.filesystem).resolve( resolved.relativePath, features: await resolved.filesystem.capabilities(), followFinal: true diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index b23825b..b22445b 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -63,7 +63,7 @@ public final class SandboxFileSystem: FileSystem, PotentialPathResolvingFileSyst await backing.capabilities() } - func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { try await backing.resolvePotentialPath(path: path) } diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index d6f1810..faa63db 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -167,7 +167,7 @@ public final class SecurityScopedFileSystem: FileSystem, PotentialPathResolvingF await backing.capabilities() } - func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { try await backing.resolvePotentialPath(path: path) } From 2e55b14d4eba7a08e84653735055a06408760e5e Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 01:17:36 -0700 Subject: [PATCH 14/34] Store checkpoints as content-addressed trees in packed blobs FileCheckpointStore moves to format v2. Each directory is a content-addressed tree of name-only, byte-sorted entries, so an incremental checkpoint writes only the trees on changed paths and identical subtrees are shared. Small blobs go into immutable pack files, synced in batches before the snapshot root publishes, and publication uses F_FULLFSYNC on Darwin where supported. Reads decode only the trees on the requested path, through a bounded LRU of decoded trees and catalog entries. The in-memory store carries content hashes over from capture instead of re-hashing snapshots. Existing v1 stores stay readable and appendable after an on-open marker migration. v2 roots use a key that v1 code cannot decode, so an old process still running during an upgrade fails closed instead of treating the new trees as empty. Compaction handles both formats, packs, interrupted relocation and orphan sweeping. On the medium benchmark fixture, the first directory checkpoint drops from 3.9 s to 1.1 s, an incremental checkpoint writes about 29 KB instead of 4.8 MB, and a warm checkpoint read takes 0.4 ms instead of 61 ms. --- Package.swift | 3 +- README.md | 54 +- .../Workspace/ChangeSet+RevisionIndex.swift | 4 +- Sources/Workspace/CheckpointStore.swift | 1135 ++++++++++++++--- Sources/Workspace/RevisionIndex.swift | 25 +- Sources/Workspace/Snapshot.swift | 14 +- Sources/Workspace/Support/CostLRU.swift | 59 + Sources/Workspace/Support/SHA256.swift | 117 +- Sources/Workspace/TextDiff.swift | 19 +- Sources/Workspace/WorkspaceRevision.swift | 77 +- .../CheckpointFormatTests.swift | 577 +++++++++ Tests/WorkspaceTests/Fixtures/README.md | 7 + ...00-0000-0000-0000-000000000001.format.lock | 0 ...cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 | 1 + ...d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 | 1 + ...aff9b70ad0b84a09286df071b9159796462b635ae2 | 1 + .../669D476B-6EFC-46E0-A1FB-8155C3DA8121.json | 33 + .../8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json | 28 + .../format.json | 3 + .../lifecycle.lock | 0 .../mutations.jsonl | 2 + .../mutations.lock | 0 .../mutations.sequence.json | 4 + .../5D7DC62F-9749-41E4-8BCE-90306A25C30C.json | 65 + .../EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json | 65 + .../PathIdentityRegressionTests.swift | 15 +- 26 files changed, 2014 insertions(+), 295 deletions(-) create mode 100644 Sources/Workspace/Support/CostLRU.swift create mode 100644 Tests/WorkspaceTests/CheckpointFormatTests.swift create mode 100644 Tests/WorkspaceTests/Fixtures/README.md create mode 100644 Tests/WorkspaceTests/Fixtures/v1/.10000000-0000-0000-0000-000000000001.format.lock create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/lifecycle.lock create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.lock create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json create mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json diff --git a/Package.swift b/Package.swift index 51d4525..47843ba 100644 --- a/Package.swift +++ b/Package.swift @@ -29,7 +29,8 @@ let package = Package( ), .testTarget( name: "WorkspaceTests", - dependencies: ["Workspace"] + dependencies: ["Workspace"], + resources: [.copy("Fixtures")] ), ] ) diff --git a/README.md b/README.md index ab43b59..f14c831 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,59 @@ Compaction rebases retained checkpoints to their nearest retained ancestor and r Directory-backed stores serialize checkpoint commits, revision reads, and compaction with a persistent lifecycle lock. Mutation history is independent and is not pruned by checkpoint retention. -Store format version 1 is retained so existing stores remain readable. Once a store contains `mutations.sequence.json`, do not use older library versions that ignore this file to write to that store, including through concurrent instances or after a downgrade: those writers can reuse retracted mutation sequence numbers. All writers must preserve the sequence high-water mark and retraction epoch. The sidecar now stores both fields; legacy bare-integer sidecars remain readable. Retractions and failed append reservations can leave sequence gaps. History refreshes invalidate cached records when another instance retracts or prunes mutations. +New directory stores use format version 2. Preparing an existing v1 store upgrades +`format.json` under the format, lifecycle, and mutation locks before publishing any new-format +checkpoint. Existing recursive manifests and loose blobs remain readable; migration does not +rewrite checkpoint history. The first new checkpoint over a v1 parent creates its directory +objects; historical v1 reads keep their original full-manifest decoding cost. New v1 operations +reject the v2 format marker. Schema-3 roots use the required `treeRoot` key instead of legacy +`root`, so an old operation admitted before migration fails decoding a new root: it cannot read +it as an empty tree, capture against it, or sweep its blobs during compaction. An admitted old +operation that only accesses legacy roots (or appends mutations) can still complete; migration +cannot cancel old code already running. Stop old processes before upgrading to avoid mixed-version +operation failures. +Do not downgrade the format marker. + +New snapshots use manifest schema 3: a small snapshot root points to SHA-256-addressed directory +objects in `trees/`. Each object contains byte-sorted immediate child names (never absolute paths), with child directories +referenced by hash. An incremental capture writes only changed directories and their ancestors; +unchanged objects remain shared. File reads decode only directories on the requested path. +Decoded trees and the write catalog share a **32 MiB estimated decoded-storage budget** per +store (24 MiB for trees, 8 MiB for the catalog). Both use O(1) LRU eviction and refresh recency on +hits. Estimates include entry, string-buffer and bookkeeping costs; this is not a process-RSS cap. +File metadata and directory listings also use path-only lookup. Full revision indexes are still materialized +when a whole-tree operation requests them. Metadata capture still walks the filesystem. + +Blobs retain their SHA-256 identities. Small captures use loose `blobs/` files; after the +first 32 new unique blobs in a capture, remaining blobs go into immutable `packs/.pack` +files with adjacent JSON hash/range indexes. Packs target 16 MiB and use a 1 MiB write buffer; +a single larger file may exceed the target. Tree file entries carry their pack/range, so reads +need no global pack-index decode. Full snapshot materialization maps at most four packs at a +time and copies only the required file ranges; those mappings do not escape the locked load. +The bounded write-side pack catalog is cached for one workspace. A constant-size `packs.generation` +token invalidates other writers' catalogs before compaction removes packs, without depending on +directory mtimes or enumerating historical pack files on warm writes. Hashes captured with in-memory +checkpoints are reused by revision indexes and unchanged files in subsequent checkpoints. + +Publication orders durable blob/pack writes and directory objects before the snapshot root, +and the snapshot root before checkpoint metadata. Files are synchronized before atomic rename, +and containing directories are synchronized before advancing publication. Tree-directory syncs +are batched at capture completion; compaction synchronizes each relocated tree before replacing +roots. On Darwin, snapshot-root and checkpoint-metadata publication additionally use `F_FULLFSYNC` +after syncing the containing directory, flushing the drive cache including preceding writes. +A successfully published checkpoint therefore survives process crashes and power loss on storage +that honors these synchronization requests. If `F_FULLFSYNC` fails other than interruption, +publication retains the completed file and directory `fsync`; power-loss protection is best-effort +on such volumes. Other POSIX platforms use file and directory `fsync`. +Failed captures can leave unreachable objects, but publish no checkpoint referencing incomplete content. Compaction +sweeps both formats, rewrites partially live packs, durably replaces affected snapshot roots, +and only then removes old objects. Interrupted compaction can be resumed; dry-run blob counts +and byte totals include reclaimable pack entries and incomplete orphan packs. + +All writers preserve the mutation sequence high-water mark and retraction epoch in +`mutations.sequence.json`; legacy bare-integer sidecars remain readable. Retractions and failed +append reservations can leave sequence gaps. History refreshes invalidate cached records when +another instance retracts or prunes mutations. Directory-backed checkpoint capture also reuses existing CAS blobs when a file's size and available modification timestamp are unchanged and the timestamp is more than two seconds older than the diff --git a/Sources/Workspace/ChangeSet+RevisionIndex.swift b/Sources/Workspace/ChangeSet+RevisionIndex.swift index 8c66f17..6143bb4 100644 --- a/Sources/Workspace/ChangeSet+RevisionIndex.swift +++ b/Sources/Workspace/ChangeSet+RevisionIndex.swift @@ -111,7 +111,7 @@ extension ChangeSet { let data = try await loader(path) let old = effect == .created ? Data() : data let new = effect == .created ? data : Data() - guard !old.contains(0), !new.contains(0), + guard !old.withUnsafeBytes({ $0.contains(0) }), !new.withUnsafeBytes({ $0.contains(0) }), let oldText = String(data: old, encoding: .utf8), let newText = String(data: new, encoding: .utf8) else { @@ -148,7 +148,7 @@ extension ChangeSet { async let old = loadBefore(oldPath) async let new = loadAfter(newPath) let (oldData, newData) = try await (old, new) - guard !oldData.contains(0), !newData.contains(0), + guard !oldData.withUnsafeBytes({ $0.contains(0) }), !newData.withUnsafeBytes({ $0.contains(0) }), let oldText = String(data: oldData, encoding: .utf8), let newText = String(data: newData, encoding: .utf8) else { diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 712be67..1b60dfc 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -19,7 +19,9 @@ protocol CheckpointStore: AnyObject, Sendable { func loadCheckpoint(id: UUID, workspaceId: UUID) async throws -> Checkpoint? func listCheckpoints(workspaceId: UUID) async throws -> [Checkpoint] func loadSnapshot(id: UUID, workspaceId: UUID) async throws -> Snapshot? + func resolveSnapshotFile(id: UUID, workspaceId: UUID, path: WorkspacePath) async throws -> (path: WorkspacePath, size: UInt64) func loadRevisionIndex(id: UUID, workspaceId: UUID) async throws -> RevisionIndex? + func snapshotEntry(id: UUID, workspaceId: UUID, path: WorkspacePath, includeChildren: Bool) async throws -> RevisionIndex.Entry? func readSnapshotFile( id: UUID, workspaceId: UUID, @@ -51,6 +53,20 @@ struct MutationPage: Sendable { } extension CheckpointStore { + func resolveSnapshotFile(id: UUID, workspaceId: UUID, path: WorkspacePath) async throws -> (path: WorkspacePath, size: UInt64) { + guard let index = try await loadRevisionIndex(id: id, workspaceId: workspaceId) else { + throw WorkspaceError.revisionDataNotFound(id) + } + return try index.readableFile(at: path) + } + + func snapshotEntry(id: UUID, workspaceId: UUID, path: WorkspacePath, includeChildren: Bool) async throws -> RevisionIndex.Entry? { + guard let index = try await loadRevisionIndex(id: id, workspaceId: workspaceId) else { + throw WorkspaceError.revisionDataNotFound(id) + } + return index.entry(at: path) + } + func listMutations(workspaceId: UUID) async throws -> [Mutation] { try await listMutations(workspaceId: workspaceId, afterSequence: 0).records } @@ -96,6 +112,8 @@ actor InMemoryCheckpointStore: CheckpointStore { if let parent, parentSnapshot == nil { throw WorkspaceError.storageCorrupted("checkpoint \(parent.id) has no snapshot") } + var snapshot = snapshot + snapshot.entry = Self.carryContentIDs(snapshot.entry, reusing: parentSnapshot?.entry) let checkpoint = Checkpoint.make( snapshot: snapshot, draft: draft, @@ -110,6 +128,26 @@ actor InMemoryCheckpointStore: CheckpointStore { return checkpoint } + private static func carryContentIDs(_ entry: Snapshot.Entry, reusing previous: Snapshot.Entry?) -> Snapshot.Entry { + switch entry { + case var .file(file): + if file.contentID == nil { + if case let .file(old) = previous, old.data == file.data, let hash = old.contentID { + file.contentID = hash + } else { file.contentID = SHA256.hexDigest(of: file.data) } + } + return .file(file) + case var .directory(directory): + let oldChildren: [WorkspacePath: Snapshot.Entry] + if case let .directory(old) = previous { + oldChildren = Dictionary(uniqueKeysWithValues: old.children.map { ($0.path, $0) }) + } else { oldChildren = [:] } + directory.children = directory.children.map { carryContentIDs($0, reusing: oldChildren[$0.path]) } + return .directory(directory) + case .missing, .symlink: return entry + } + } + func loadCheckpoint(id: UUID, workspaceId: UUID) async throws -> Checkpoint? { checkpointsByWorkspace[workspaceId]?.first { $0.id == id } } @@ -265,7 +303,7 @@ actor InMemoryCheckpointStore: CheckpointStore { private func collectBlobStatistics(_ entry: Snapshot.Entry, into blobs: inout [String: UInt64]) { switch entry { case let .file(file): - blobs[SHA256.hexDigest(of: file.data)] = UInt64(file.data.count) + blobs[file.contentID ?? SHA256.hexDigest(of: file.data)] = UInt64(file.data.count) case let .directory(directory): for child in directory.children { collectBlobStatistics(child, into: &blobs) } case .missing, .symlink: @@ -306,17 +344,23 @@ actor FileCheckpointStore: CheckpointStore { init( rootDirectory: URL, fileManager: FileManager = .default, - checkpointCapture: Workspace.CheckpointCapturePolicy = .trustMetadata + checkpointCapture: Workspace.CheckpointCapturePolicy = .trustMetadata, + cacheByteLimit: Int = FileCheckpointStore.defaultCacheByteLimit ) { self.rootDirectory = rootDirectory.standardizedFileURL self.fileManager = fileManager self.checkpointCapture = checkpointCapture + let budget = max(0, cacheByteLimit) + let catalogBudget = budget / 4 + self.treeCache = CostLRU(limit: budget - catalogBudget) + self.packCatalog = CostLRU(limit: catalogBudget) let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys] self.encoder = encoder self.decoder = JSONDecoder() self.compactEncoder = JSONEncoder() + self.compactEncoder.outputFormatting = [.sortedKeys] } func prepare(workspaceId: UUID) async throws { @@ -335,11 +379,15 @@ actor FileCheckpointStore: CheckpointStore { let mutationCursor = try await persistedMutationCursor(workspaceId: draft.workspaceID) let checkpoints = try loadAllCheckpointsUnlocked(workspaceId: draft.workspaceID) let parent = resolvedParent(for: draft, checkpoints: checkpoints) - let parentSnapshot = try parent.map { - guard let snapshot = try loadSnapshotUnlocked(id: $0.snapshotId, workspaceId: draft.workspaceID) else { + let parentManifest = try parent.map { + guard let manifest = try loadManifestUnlocked(id: $0.snapshotId, workspaceId: draft.workspaceID) else { throw WorkspaceError.storageCorrupted("checkpoint \($0.id) has no snapshot") } - return snapshot + return manifest + } + let parentSnapshot = try parentManifest.map { + Snapshot(id: $0.id, rootPath: $0.rootPath, + entry: try snapshotEntry(from: $0.root, workspaceId: draft.workspaceID)) } let checkpoint = Checkpoint.make( snapshot: snapshot, @@ -348,8 +396,8 @@ actor FileCheckpointStore: CheckpointStore { parent: parent, parentSnapshot: parentSnapshot ) - try writeSnapshotUnlocked(snapshot, workspaceId: draft.workspaceID) - try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: draft.workspaceID)) + try writeSnapshotUnlocked(snapshot, workspaceId: draft.workspaceID, reusing: parentManifest?.root) + try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: draft.workspaceID), durability: .publication) listCheckpointsCache[draft.workspaceID] = nil return checkpoint } @@ -390,11 +438,11 @@ actor FileCheckpointStore: CheckpointStore { } } - /// A persisted snapshot: the tree structure with file contents replaced by content hashes. - /// The bytes themselves live in per-workspace `blobs/` files, so identical content is - /// stored once no matter how many snapshots reference it, and never base64-inflated. + /// Schemas 1/2 inline the recursive tree and reference loose blobs. Schema 3 + /// stores a small root, content-addressed directory references, and optional pack ranges. + /// The outer store format is version 2, fencing writers that only understand inline trees. private struct SnapshotManifest: Codable, Sendable { - struct Node: Codable, Sendable { + struct Node: Codable, Sendable, Equatable { enum Kind: String, Codable { case file, directory, symlink, missing } @@ -407,6 +455,16 @@ actor FileCheckpointStore: CheckpointStore { var modificationDate: Date? = nil var symlinkTarget: String? var children: [Node]? + var treeHash: String? = nil + var packed: PackedBlob? = nil + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.kind == rhs.kind && lhs.path == rhs.path && + lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && + lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && + lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && + lhs.children == rhs.children && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed + } } var version: Int @@ -415,6 +473,87 @@ actor FileCheckpointStore: CheckpointStore { var root: Node // Older manifests and materialized snapshots have no trustworthy capture timestamp. var captureStartedAt: Date? = nil + + private enum CodingKeys: String, CodingKey { + case version, id, rootPath, root, treeRoot, captureStartedAt + } + + init(version: Int, id: UUID, rootPath: WorkspacePath, root: Node, captureStartedAt: Date? = nil) { + self.version = version + self.id = id + self.rootPath = rootPath + self.root = root + self.captureStartedAt = captureStartedAt + } + + init(from decoder: any Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + version = try values.decode(Int.self, forKey: .version) + id = try values.decode(UUID.self, forKey: .id) + rootPath = try values.decode(WorkspacePath.self, forKey: .rootPath) + root = try values.decode(Node.self, forKey: version >= 3 ? .treeRoot : .root) + captureStartedAt = try values.decodeIfPresent(Date.self, forKey: .captureStartedAt) + } + + func encode(to encoder: any Encoder) throws { + var values = encoder.container(keyedBy: CodingKeys.self) + try values.encode(version, forKey: .version) + try values.encode(id, forKey: .id) + try values.encode(rootPath, forKey: .rootPath) + // Deliberately omit v1's required root key: even an already-admitted + // legacy operation must fail decoding before it can sweep live blobs. + try values.encode(root, forKey: version >= 3 ? .treeRoot : .root) + try values.encodeIfPresent(captureStartedAt, forKey: .captureStartedAt) + } + } + + /// Names are validated before entering the cache; no absolute paths are persisted. + private struct TreeEntry: Codable, Sendable { + var name: String + var kind: SnapshotManifest.Node.Kind + var permissions: POSIXPermissions? + var contentHash: String? + var contentSize: UInt64? + var modificationDate: Date? + var symlinkTarget: String? + var treeHash: String? + var packed: PackedBlob? + + init(_ node: SnapshotManifest.Node) throws { + name = node.path.name + try Self.validate(name) + kind = node.kind + permissions = node.permissions + contentHash = node.contentHash + contentSize = node.contentSize + modificationDate = node.modificationDate + symlinkTarget = node.symlinkTarget + treeHash = node.treeHash + packed = node.packed + } + + static func validate(_ name: String) throws { + guard !name.isEmpty, !name.utf8.elementsEqual(".".utf8), !name.utf8.elementsEqual("..".utf8), + !name.utf8.contains(47), !name.utf8.contains(0) else { + throw WorkspaceError.storageCorrupted("invalid tree entry name") + } + } + + func node(in parent: WorkspacePath) -> SnapshotManifest.Node { + .init(kind: kind, path: parent.appending(name), permissions: permissions, + contentHash: contentHash, contentSize: contentSize, modificationDate: modificationDate, + symlinkTarget: symlinkTarget, treeHash: treeHash, packed: packed) + } + + var estimatedBytes: Int { + // String buffers, array storage, optional fields and allocator overhead. + var stringBytes = name.utf8.count + stringBytes += contentHash?.utf8.count ?? 0 + stringBytes += treeHash?.utf8.count ?? 0 + stringBytes += symlinkTarget?.utf8.count ?? 0 + stringBytes += packed?.pack.utf8.count ?? 0 + return 256 + 2 * stringBytes + } } func captureRevision(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint { @@ -435,6 +574,10 @@ actor FileCheckpointStore: CheckpointStore { } else { parentManifest = nil } + try self.requireCurrentFormatUnlocked(workspaceId: draft.workspaceID) + let batch = try self.makeBlobBatch(workspaceId: draft.workspaceID) + defer { batch.close() } + batch.summarizeInsertions = parentManifest == nil let captureStartedAt = Date() let snapshotID = UUID() let root = try await self.captureManifestNode( @@ -442,69 +585,57 @@ actor FileCheckpointStore: CheckpointStore { at: .root, reusing: parentManifest?.root, previousCaptureStartedAt: parentManifest?.captureStartedAt, - workspaceId: draft.workspaceID + workspaceId: draft.workspaceID, + batch: batch ) + try self.finish(batch, workspaceId: draft.workspaceID) let manifest = SnapshotManifest( - version: 2, id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt - ) - let beforeIndex = try parentManifest.map { - RevisionIndex( - id: $0.id, - root: $0.rootPath, - entry: try self.revisionEntry(from: $0.root, workspaceId: draft.workspaceID) - ) - } ?? RevisionIndex( - id: UUID(), - root: .root, - entry: .missing(path: .root) - ) - let afterIndex = RevisionIndex( - id: snapshotID, - root: .root, - entry: try self.revisionEntry(from: root, workspaceId: draft.workspaceID) - ) - let blobsRoot = self.blobsDirectoryURL(workspaceId: draft.workspaceID) - let beforeRoot = parentManifest?.root - let changes = try await ChangeSet.compare( - before: beforeIndex, - after: afterIndex, - maxTextBytes: 1_000_000, - loadBefore: { path in try Self.loadManifestFile(path, root: beforeRoot, blobsRoot: blobsRoot) }, - loadAfter: { path in try Self.loadManifestFile(path, root: root, blobsRoot: blobsRoot) } + version: 3, id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt ) + let summary: ChangeSet.Summary + if let parentManifest { + summary = try await self.compareNodes(parentManifest.root, root, workspaceId: draft.workspaceID).summary + } else { + summary = try self.creationSummary(root, workspaceId: draft.workspaceID, insertedLines: batch.insertedLines) + } let checkpoint = Checkpoint.make( snapshotID: snapshotID, draft: draft, mutationCursor: mutationCursor, parent: parent, - summary: changes.summary + summary: summary ) - try self.write(manifest, to: self.snapshotURL(id: snapshotID, workspaceId: draft.workspaceID)) + try Task.checkCancellation() + try self.write(manifest, to: self.snapshotURL(id: snapshotID, workspaceId: draft.workspaceID), durability: .publication) try self.write( checkpoint, - to: self.checkpointURL(id: checkpoint.id, workspaceId: draft.workspaceID) + to: self.checkpointURL(id: checkpoint.id, workspaceId: draft.workspaceID), durability: .publication ) self.listCheckpointsCache[draft.workspaceID] = nil return checkpoint } } - private func writeSnapshotUnlocked(_ snapshot: Snapshot, workspaceId: UUID) throws { - let root = try writeManifestNode(snapshot.entry, workspaceId: workspaceId) + private func writeSnapshotUnlocked(_ snapshot: Snapshot, workspaceId: UUID, reusing previous: SnapshotManifest.Node?) throws { + try requireCurrentFormatUnlocked(workspaceId: workspaceId) + let batch = try makeBlobBatch(workspaceId: workspaceId) + defer { batch.close() } + let root = try writeManifestNode(snapshot.entry, workspaceId: workspaceId, batch: batch, reusing: previous) + try finish(batch, workspaceId: workspaceId) let manifest = SnapshotManifest( - version: 2, + version: 3, id: snapshot.id, rootPath: snapshot.rootPath, root: root ) - try write(manifest, to: snapshotURL(id: snapshot.id, workspaceId: workspaceId)) + try write(manifest, to: snapshotURL(id: snapshot.id, workspaceId: workspaceId), durability: .publication) } private func loadManifestUnlocked(id: UUID, workspaceId: UUID) throws -> SnapshotManifest? { let url = snapshotURL(id: id, workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return nil } let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) - guard manifest.id == id else { + guard (1...3).contains(manifest.version), manifest.id == id else { throw WorkspaceError.storageCorrupted("snapshot manifest id does not match \(id)") } return manifest @@ -515,8 +646,10 @@ actor FileCheckpointStore: CheckpointStore { at path: WorkspacePath, reusing previous: SnapshotManifest.Node?, previousCaptureStartedAt: Date?, - workspaceId: UUID + workspaceId: UUID, + batch: BlobBatch ) async throws -> SnapshotManifest.Node { + try Task.checkCancellation() // exists follows the final symlink on some backends, losing dangling // links and cycles. Capture the entry itself, just as Snapshot.capture does. guard let info = try await Snapshot.infoIfPresent(at: path, on: filesystem) else { @@ -536,31 +669,38 @@ actor FileCheckpointStore: CheckpointStore { previous?.modificationDate == info.modificationDate, let hash = previous?.contentHash, Self.isValidContentHash(hash), - fileManager.fileExists(atPath: blobURL(hash: hash, workspaceId: workspaceId).path) { + let previous, + blobExists(previous, workspaceId: workspaceId, batch: batch) { return SnapshotManifest.Node( kind: .file, path: path, permissions: info.permissions, contentHash: hash, contentSize: info.size, - modificationDate: info.modificationDate + modificationDate: info.modificationDate, + packed: previous.packed ) } let data = try await filesystem.readFile(path: path) let hash = SHA256.hexDigest(of: data) - let blob = blobURL(hash: hash, workspaceId: workspaceId) - if !fileManager.fileExists(atPath: blob.path) { try data.write(to: blob, options: .atomic) } + if batch.summarizeInsertions, data.count <= 1_000_000, + !data.withUnsafeBytes({ $0.contains(0) }), let text = String(data: data, encoding: .utf8) { + batch.insertedLines[path] = TextDiff.lineBased(from: "", to: text).statistics.additions + } + let packed = try batch.store(data, hash: hash) return SnapshotManifest.Node( kind: .file, path: path, permissions: info.permissions, contentHash: hash, contentSize: UInt64(data.count), - modificationDate: info.modificationDate + modificationDate: info.modificationDate, + packed: packed ) case .directory: var previousChildren: [PathName: SnapshotManifest.Node] = [:] - for child in previous?.kind == .directory ? previous?.children ?? [] : [] { + let previousEntries = try previous.map { try self.children(of: $0, workspaceId: workspaceId) } ?? [] + for child in previousEntries { guard previousChildren.updateValue(child, forKey: PathName(child.path.name)) == nil else { throw WorkspaceError.storageCorrupted( "snapshot directory \(path) contains duplicate child \(child.path.name)" @@ -569,23 +709,32 @@ actor FileCheckpointStore: CheckpointStore { } var children: [SnapshotManifest.Node] = [] for entry in try await filesystem.listDirectory(path: path).sorted(by: { PathName($0.name) < PathName($1.name) }) { + try TreeEntry.validate(entry.name) children.append( try await captureManifestNode( from: filesystem, at: path.appending(entry.name), reusing: previousChildren[PathName(entry.name)], previousCaptureStartedAt: previousCaptureStartedAt, - workspaceId: workspaceId + workspaceId: workspaceId, + batch: batch ) ) } - return SnapshotManifest.Node( + if let previous, previous.kind == .directory, previous.treeHash != nil, + previousEntries == children { + var reused = previous + reused.permissions = info.permissions + reused.modificationDate = info.modificationDate + return reused + } + return try persistDirectory(SnapshotManifest.Node( kind: .directory, path: path, permissions: info.permissions, modificationDate: info.modificationDate, children: children - ) + ), workspaceId: workspaceId) case .symlink: return SnapshotManifest.Node( kind: .symlink, @@ -597,32 +746,6 @@ actor FileCheckpointStore: CheckpointStore { } } - private static func loadManifestFile( - _ path: WorkspacePath, - root: SnapshotManifest.Node?, - blobsRoot: URL - ) throws -> Data { - guard let root, let node = manifestNodeStatic(at: path, in: root), node.kind == .file, - let hash = node.contentHash, isValidContentHash(hash) - else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } - let url = blobsRoot.appendingPathComponent(hash, isDirectory: false) - guard FileManager.default.fileExists(atPath: url.path) else { - throw WorkspaceError.storageCorrupted("missing content blob \(hash) for \(path)") - } - return try Data(contentsOf: url) - } - - private static func manifestNodeStatic( - at path: WorkspacePath, - in node: SnapshotManifest.Node - ) -> SnapshotManifest.Node? { - if node.path == path { return node } - for child in node.children ?? [] where path.isWithin(child.path) { - return manifestNodeStatic(at: path, in: child) - } - return nil - } - func loadSnapshot(id: UUID, workspaceId: UUID) async throws -> Snapshot? { try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } @@ -635,12 +758,7 @@ actor FileCheckpointStore: CheckpointStore { try await validateWorkspaceFormatIfPresent(workspaceId) guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { - let url = snapshotURL(id: id, workspaceId: workspaceId) - guard fileManager.fileExists(atPath: url.path) else { return nil } - let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) - guard manifest.id == id else { - throw WorkspaceError.storageCorrupted("snapshot manifest id does not match \(id)") - } + guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { return nil } return RevisionIndex( id: manifest.id, root: manifest.rootPath, @@ -662,33 +780,17 @@ actor FileCheckpointStore: CheckpointStore { } guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { - let url = snapshotURL(id: id, workspaceId: workspaceId) - guard fileManager.fileExists(atPath: url.path) else { return nil } - let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) - guard manifest.id == id else { - throw WorkspaceError.storageCorrupted("snapshot manifest id does not match \(id)") - } - guard let node = manifestNode(at: path, in: manifest.root), node.kind == .file else { return nil } - guard let hash = node.contentHash, Self.isValidContentHash(hash) else { - throw WorkspaceError.storageCorrupted("snapshot file node \(node.path) has an invalid content hash") - } - let blob = blobURL(hash: hash, workspaceId: workspaceId) - guard fileManager.fileExists(atPath: blob.path) else { - throw WorkspaceError.storageCorrupted("missing content blob \(hash) for \(path)") - } - let handle = try FileHandle(forReadingFrom: blob) - defer { try? handle.close() } - let size = try handle.seekToEnd() - guard offset < size else { return Data() } - try handle.seek(toOffset: offset) - let count = min(UInt64(length ?? Int.max), size - offset) - return try handle.read(upToCount: Int(count)) ?? Data() + guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId), + let node = try manifestNode(at: path, in: manifest.root, workspaceId: workspaceId), node.kind == .file + else { return nil } + return try readBlob(node, workspaceId: workspaceId, offset: offset, length: length) } } private func revisionEntry( from node: SnapshotManifest.Node, - workspaceId: UUID + workspaceId: UUID, + depth: Int? = nil ) throws -> RevisionIndex.Entry { switch node.kind { case .missing: @@ -707,7 +809,9 @@ actor FileCheckpointStore: CheckpointStore { return .directory( path: node.path, permissions: node.permissions ?? .defaultDirectory, - children: try (node.children ?? []).map { try revisionEntry(from: $0, workspaceId: workspaceId) } + children: depth == 0 ? [] : try children(of: node, workspaceId: workspaceId).map { + try revisionEntry(from: $0, workspaceId: workspaceId, depth: depth.map { $0 - 1 }) + } ) case .symlink: return .symbolicLink( @@ -718,56 +822,88 @@ actor FileCheckpointStore: CheckpointStore { } } - private func manifestNode(at path: WorkspacePath, in node: SnapshotManifest.Node) -> SnapshotManifest.Node? { - if node.path == path { return node } - for child in node.children ?? [] where path.isWithin(child.path) { - return manifestNode(at: path, in: child) + private func manifestNode(at path: WorkspacePath, in root: SnapshotManifest.Node, workspaceId: UUID) throws -> SnapshotManifest.Node? { + let components = path.components + guard path.isWithin(root.path) else { return nil } + let prefix = root.path.components + var node = root + for component in components.dropFirst(prefix.count) { + let name = PathName(component) + guard node.kind == .directory else { return nil } + if let hash = node.treeHash { + let entries = try treeEntries(hash: hash, workspaceId: workspaceId) + var low = 0, high = entries.count + while low < high { + let mid = (low + high) / 2 + if PathName(entries[mid].name) < name { low = mid + 1 } else { high = mid } + } + guard low < entries.count, PathName(entries[low].name) == name else { return nil } + node = entries[low].node(in: node.path) + } else { + // Legacy manifests already contain absolute child nodes. + guard let child = node.children?.first(where: { PathName($0.path.name) == name }) else { return nil } + node = child + } } - return nil + return node } private func writeManifestNode( _ entry: Snapshot.Entry, - workspaceId: UUID + workspaceId: UUID, + batch: BlobBatch, + reusing previous: SnapshotManifest.Node? ) throws -> SnapshotManifest.Node { switch entry { case let .missing(missing): return SnapshotManifest.Node(kind: .missing, path: missing.path) case let .file(file): - let hash = SHA256.hexDigest(of: file.data) - let url = blobURL(hash: hash, workspaceId: workspaceId) - if !fileManager.fileExists(atPath: url.path) { - try file.data.write(to: url, options: .atomic) + let hash = file.contentID ?? SHA256.hexDigest(of: file.data) + if let previous, previous.kind == .file, previous.path == file.path, + previous.contentHash == hash, previous.contentSize == UInt64(file.data.count) { + var reused = previous + reused.permissions = file.permissions + return reused } + let packed = try batch.store(file.data, hash: hash) return SnapshotManifest.Node( kind: .file, path: file.path, permissions: file.permissions, contentHash: hash, - contentSize: UInt64(file.data.count) + contentSize: UInt64(file.data.count), + packed: packed ) case let .symlink(symlink): - return SnapshotManifest.Node( - kind: .symlink, - path: symlink.path, - permissions: symlink.permissions, - symlinkTarget: symlink.target - ) + if let previous, previous.kind == .symlink, previous.path == symlink.path, + previous.symlinkTarget.map(PathName.init) == PathName(symlink.target) { + var reused = previous + reused.permissions = symlink.permissions + return reused + } + return SnapshotManifest.Node(kind: .symlink, path: symlink.path, + permissions: symlink.permissions, symlinkTarget: symlink.target) case let .directory(directory): - return SnapshotManifest.Node( - kind: .directory, - path: directory.path, - permissions: directory.permissions, - children: try directory.children.map { - try writeManifestNode($0, workspaceId: workspaceId) - } - ) + let old = try previous.map { try children(of: $0, workspaceId: workspaceId) } ?? [] + let byPath = Dictionary(uniqueKeysWithValues: old.map { ($0.path, $0) }) + let entries = try directory.children.map { + try writeManifestNode($0, workspaceId: workspaceId, batch: batch, reusing: byPath[$0.path]) + }.sorted { PathName($0.path.name) < PathName($1.path.name) } + if let previous, previous.kind == .directory, previous.treeHash != nil, old == entries { + var reused = previous + reused.permissions = directory.permissions + return reused + } + return try persistDirectory(SnapshotManifest.Node( + kind: .directory, path: directory.path, permissions: directory.permissions, children: entries + ), workspaceId: workspaceId) } } private func snapshotEntry( from node: SnapshotManifest.Node, - workspaceId: UUID + workspaceId: UUID, + cache: SnapshotBlobCache = SnapshotBlobCache() ) throws -> Snapshot.Entry { switch node.kind { case .missing: @@ -776,15 +912,12 @@ actor FileCheckpointStore: CheckpointStore { guard let hash = node.contentHash, Self.isValidContentHash(hash) else { throw WorkspaceError.storageCorrupted("snapshot file node \(node.path) has an invalid content hash") } - let url = blobURL(hash: hash, workspaceId: workspaceId) - guard fileManager.fileExists(atPath: url.path) else { - throw WorkspaceError.storageCorrupted("missing content blob \(hash) for \(node.path)") - } return .file( Snapshot.File( path: node.path, - data: try Data(contentsOf: url), - permissions: node.permissions ?? POSIXPermissions(0o644) + data: try materializedBlob(node, workspaceId: workspaceId, cache: cache), + permissions: node.permissions ?? POSIXPermissions(0o644), + contentID: hash ) ) case .symlink: @@ -800,8 +933,8 @@ actor FileCheckpointStore: CheckpointStore { Snapshot.Directory( path: node.path, permissions: node.permissions ?? POSIXPermissions(0o755), - children: try (node.children ?? []).map { - try snapshotEntry(from: $0, workspaceId: workspaceId) + children: try children(of: node, workspaceId: workspaceId).map { + try snapshotEntry(from: $0, workspaceId: workspaceId, cache: cache) } ) ) @@ -822,7 +955,7 @@ actor FileCheckpointStore: CheckpointStore { state.highWaterMark += 1 record.sequence = state.highWaterMark // Reserve before publishing. Failed appends and retractions cannot reuse an ID. - try write(state, to: mutationSequenceURL(workspaceId: mutation.workspaceID)) + try write(state, to: mutationSequenceURL(workspaceId: mutation.workspaceID), durability: .atomic) if fileManager.fileExists(atPath: jsonl.path) { try appendJSONLLine(encode: record, to: jsonl) @@ -841,7 +974,7 @@ actor FileCheckpointStore: CheckpointStore { let records = fileManager.fileExists(atPath: url.path) ? try loadMutationsFromJSONL(at: url) : [] try state.bumpRetractionEpoch() // Publish invalidation before rewriting. A failed rewrite only causes a harmless reload. - try write(state, to: mutationSequenceURL(workspaceId: workspaceId)) + try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .atomic) try writeAllMutationsAsJSONL(records.filter { $0.sequence != sequence }, to: url) } } @@ -884,7 +1017,7 @@ actor FileCheckpointStore: CheckpointStore { return } try state.bumpRetractionEpoch() - try write(state, to: mutationSequenceURL(workspaceId: workspaceId)) + try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .atomic) try writeAllMutationsAsJSONL(kept, to: jsonl) } } @@ -938,6 +1071,9 @@ actor FileCheckpointStore: CheckpointStore { let retainedSnapshotIDs = Set(retained.map(\.snapshotId)) var retainedBlobHashes: Set = [] + var retainedTrees: Set = [] + var retainedPacks: Set = [] + var livePacked: [String: PackedBlob] = [:] for checkpoint in retained { let manifestURL = snapshotURL(id: checkpoint.snapshotId, workspaceId: workspaceId) guard fileManager.fileExists(atPath: manifestURL.path) else { @@ -947,7 +1083,8 @@ actor FileCheckpointStore: CheckpointStore { guard manifest.id == checkpoint.snapshotId else { throw WorkspaceError.storageCorrupted("checkpoint \(checkpoint.id) references a mismatched snapshot") } - try collectContentHashes(from: manifest.root, into: &retainedBlobHashes) + try collectArtifacts(from: manifest.root, workspaceId: workspaceId, blobs: &retainedBlobHashes, + trees: &retainedTrees, packs: &retainedPacks, livePacked: &livePacked) } var retainedBlobBytes: UInt64 = 0 @@ -958,10 +1095,11 @@ actor FileCheckpointStore: CheckpointStore { } retainedBlobBytes += try fileSize(at: url) } + retainedBlobBytes += livePacked.values.reduce(0) { $0 + $1.size } let after = Workspace.StorageStatistics( checkpointCount: retained.count, snapshotCount: retainedSnapshotIDs.count, - blobCount: retainedBlobHashes.count, + blobCount: retainedBlobHashes.count + livePacked.count, blobBytes: retainedBlobBytes ) let report = Workspace.CompactionReport( @@ -974,12 +1112,33 @@ actor FileCheckpointStore: CheckpointStore { guard !dryRun else { return StoreCompactionResult(report: report, checkpoints: retained) } + // Rewrite partial packs before changing any references. Then atomically replace + // retained snapshot roots; a crash at any point leaves old and/or new packs live. + let relocated = try repackLiveBlobs(livePacked, workspaceId: workspaceId) + if !relocated.isEmpty { + retainedTrees.removeAll() + retainedPacks.removeAll() + for id in retainedSnapshotIDs { + guard var manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { + throw WorkspaceError.storageCorrupted("missing retained snapshot") + } + let root = try relocating(manifest.root, blobs: relocated, workspaceId: workspaceId) + if root != manifest.root { + manifest.root = root + manifest.version = 3 + try write(manifest, to: snapshotURL(id: id, workspaceId: workspaceId), durability: .publication) + } + try collectArtifacts(from: manifest.root, workspaceId: workspaceId, blobs: &retainedBlobHashes, + trees: &retainedTrees, packs: &retainedPacks, livePacked: &livePacked) + } + } for checkpoint in retained where rebasedIDs.contains(checkpoint.id) { - try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: workspaceId)) + try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: workspaceId), durability: .publication) } for id in plan.removedCheckpointIDs { try removeIfPresent(checkpointURL(id: id, workspaceId: workspaceId)) } + try Self.syncDirectory(checkpointsDirectoryURL(workspaceId: workspaceId)) for url in try artifactURLs(in: snapshotsDirectoryURL(workspaceId: workspaceId), pathExtension: "json") { guard let id = UUID(uuidString: url.deletingPathExtension().lastPathComponent), retainedSnapshotIDs.contains(id) @@ -988,10 +1147,40 @@ actor FileCheckpointStore: CheckpointStore { continue } } + try Self.syncDirectory(snapshotsDirectoryURL(workspaceId: workspaceId)) + // Invalidate other writers' catalogs durably before deleting any old pack. + if fileManager.fileExists(atPath: packsDirectoryURL(workspaceId: workspaceId).path) { + try advancePackGeneration(workspaceId: workspaceId) + } + for (directory, retainedNames) in [ + (treesDirectoryURL(workspaceId: workspaceId), retainedTrees), + (packsDirectoryURL(workspaceId: workspaceId), Set(retainedPacks.flatMap { [$0 + ".pack", $0 + ".json"] })) + ] { + if fileManager.fileExists(atPath: directory.path) { + for url in try fileManager.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + where !retainedNames.contains(url.lastPathComponent) { + try fileManager.removeItem(at: url) + } + try Self.syncDirectory(directory) + } + } + treeCache.removeAll() + packCatalog.removeAll() + packCatalogKey = nil for url in try blobURLs(workspaceId: workspaceId) where !retainedBlobHashes.contains(url.lastPathComponent) { try fileManager.removeItem(at: url) } + for directory in [ + workspaceDirectoryURL(workspaceId: workspaceId), checkpointsDirectoryURL(workspaceId: workspaceId), + snapshotsDirectoryURL(workspaceId: workspaceId), blobsDirectoryURL(workspaceId: workspaceId) + ] { + for url in try fileManager.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + where url.lastPathComponent.hasPrefix(".pending-") { + try fileManager.removeItem(at: url) + } + } + try Self.syncDirectory(blobsDirectoryURL(workspaceId: workspaceId)) listCheckpointsCache[workspaceId] = nil return StoreCompactionResult(report: report, checkpoints: retained) } @@ -1011,12 +1200,7 @@ actor FileCheckpointStore: CheckpointStore { } private func loadSnapshotUnlocked(id: UUID, workspaceId: UUID) throws -> Snapshot? { - let url = snapshotURL(id: id, workspaceId: workspaceId) - guard fileManager.fileExists(atPath: url.path) else { return nil } - let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) - guard manifest.id == id else { - throw WorkspaceError.storageCorrupted("snapshot manifest id does not match \(id)") - } + guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { return nil } return Snapshot( id: manifest.id, rootPath: manifest.rootPath, @@ -1046,21 +1230,27 @@ actor FileCheckpointStore: CheckpointStore { ) let blobs = try blobURLs(workspaceId: workspaceId) var blobBytes: UInt64 = 0 + var packedCount = 0 for blob in blobs { blobBytes += try fileSize(at: blob) } + for url in try artifactURLs(in: packsDirectoryURL(workspaceId: workspaceId), pathExtension: "pack", includeHidden: true) { + let usage = try packStatistics(url.deletingPathExtension().lastPathComponent, workspaceId: workspaceId) + blobBytes += usage.bytes + packedCount += usage.count + } return Workspace.StorageStatistics( checkpointCount: checkpoints.count, snapshotCount: snapshots.count, - blobCount: blobs.count, + blobCount: blobs.count + packedCount, blobBytes: blobBytes ) } - private func artifactURLs(in directory: URL, pathExtension: String) throws -> [URL] { + private func artifactURLs(in directory: URL, pathExtension: String, includeHidden: Bool = false) throws -> [URL] { guard fileManager.fileExists(atPath: directory.path) else { return [] } return try fileManager.contentsOfDirectory( at: directory, includingPropertiesForKeys: nil, - options: [.skipsHiddenFiles] + options: includeHidden ? [] : [.skipsHiddenFiles] ).filter { $0.pathExtension == pathExtension }.map { url in let type = try fileManager.attributesOfItem(atPath: url.path)[.type] as? FileAttributeType guard type == .typeRegular else { @@ -1076,7 +1266,7 @@ actor FileCheckpointStore: CheckpointStore { return try fileManager.contentsOfDirectory( at: directory, includingPropertiesForKeys: nil, - options: [.skipsHiddenFiles] + options: [] ).map { url in let type = try fileManager.attributesOfItem(atPath: url.path)[.type] as? FileAttributeType guard type == .typeRegular else { @@ -1091,18 +1281,87 @@ actor FileCheckpointStore: CheckpointStore { return value?.uint64Value ?? 0 } - private func collectContentHashes(from node: SnapshotManifest.Node, into hashes: inout Set) throws { + private func packStatistics(_ pack: String, workspaceId: UUID) throws -> (count: Int, bytes: UInt64) { + let directory = packsDirectoryURL(workspaceId: workspaceId) + let index = try? read([String: PackedBlob].self, from: directory.appendingPathComponent(pack + ".json")) + return (index?.count ?? 1, try fileSize(at: directory.appendingPathComponent(pack + ".pack"))) + } + + private func collectArtifacts(from node: SnapshotManifest.Node, workspaceId: UUID, + blobs: inout Set, trees: inout Set, packs: inout Set, + livePacked: inout [String: PackedBlob]) throws { switch node.kind { case .file: - guard let hash = node.contentHash, Self.isValidContentHash(hash) else { - throw WorkspaceError.storageCorrupted("snapshot file node \(node.path) has an invalid content hash") + let location = try blobLocation(node, workspaceId: workspaceId) + guard fileManager.fileExists(atPath: location.url.path), + try fileSize(at: location.url) >= location.offset + location.size else { + throw WorkspaceError.storageCorrupted("missing or truncated blob for \(node.path)") + } + if let packed = node.packed, let hash = node.contentHash { + packs.insert(packed.pack) + livePacked[hash] = packed + } else if let hash = node.contentHash { + blobs.insert(hash) } - hashes.insert(hash) case .directory: - for child in node.children ?? [] { try collectContentHashes(from: child, into: &hashes) } - case .missing, .symlink: - break + if let hash = node.treeHash, !trees.insert(hash).inserted { return } + for child in try children(of: node, workspaceId: workspaceId) { + try collectArtifacts(from: child, workspaceId: workspaceId, blobs: &blobs, trees: &trees, packs: &packs, livePacked: &livePacked) + } + case .missing, .symlink: break + } + } + + private func repackLiveBlobs(_ live: [String: PackedBlob], workspaceId: UUID) throws -> [String: PackedBlob] { + let groups = Dictionary(grouping: live.keys, by: { live[$0]!.pack }) + // A previous compaction may have stopped halfway through root replacement. + // Canonicalize every live hash, even when its chosen pack needs no rewriting. + var relocated = live + let directory = packsDirectoryURL(workspaceId: workspaceId) + for (pack, hashes) in groups { + let oldIndex = try? read([String: PackedBlob].self, from: directory.appendingPathComponent(pack + ".json")) + if let oldIndex, oldIndex.count == hashes.count, + hashes.allSatisfy({ oldIndex[$0] == live[$0] }), + try fileSize(at: directory.appendingPathComponent(pack + ".pack")) == hashes.reduce(UInt64(0), { $0 + live[$1]!.size }) { + continue + } + let id = UUID().uuidString + var bytes = Data() + var index: [String: PackedBlob] = [:] + let handle = try FileHandle(forReadingFrom: directory.appendingPathComponent(pack + ".pack")) + defer { try? handle.close() } + for hash in hashes.sorted() { + let old = live[hash]! + guard old.size <= UInt64(Int.max) else { throw WorkspaceError.storageCorrupted("oversized packed blob") } + try handle.seek(toOffset: old.offset) + let content = try handle.read(upToCount: Int(old.size)) ?? Data() + guard content.count == Int(old.size), SHA256.hexDigest(of: content) == hash else { + throw WorkspaceError.storageCorrupted("invalid packed content during compaction") + } + let replacement = PackedBlob(pack: id, offset: UInt64(bytes.count), size: old.size) + index[hash] = replacement + relocated[hash] = replacement + bytes.append(content) + } + try Self.durableWrite(bytes, to: directory.appendingPathComponent(id + ".pack")) + try Self.durableWrite(compactEncoder.encode(index), to: directory.appendingPathComponent(id + ".json")) + } + return relocated + } + + private func relocating(_ node: SnapshotManifest.Node, blobs: [String: PackedBlob], workspaceId: UUID) throws -> SnapshotManifest.Node { + var result = node + if node.kind == .file, let hash = node.contentHash, node.packed != nil, let replacement = blobs[hash] { + result.packed = replacement + } else if node.kind == .directory { + let old = try children(of: node, workspaceId: workspaceId) + let new = try old.map { try relocating($0, blobs: blobs, workspaceId: workspaceId) } + if old != new { + result.children = new + result = try persistDirectory(result, workspaceId: workspaceId, syncParent: true) + } } + return result } private static func isValidContentHash(_ hash: String) -> Bool { @@ -1344,51 +1603,76 @@ actor FileCheckpointStore: CheckpointStore { return "\(mtime?.timeIntervalSince1970 ?? 0)-\(count)" } + private func ensureDirectory(_ url: URL) throws { + var isDirectory: ObjCBool = false + if fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory), isDirectory.boolValue { return } + try fileManager.createDirectory(at: url, withIntermediateDirectories: true) + } + private func ensureWorkspaceDirectories(for workspaceId: UUID) async throws { - try fileManager.createDirectory(at: rootDirectory, withIntermediateDirectories: true) + try ensureDirectory(rootDirectory) try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { - try ensureWorkspaceDirectoriesLocked(for: workspaceId) + let version = try ensureWorkspaceDirectoriesLocked(for: workspaceId) + if version == 1 { + try await withExclusiveLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + try upgradeFormatUnlocked(workspaceId: workspaceId) + } + } + } } } - private func ensureWorkspaceDirectoriesLocked(for workspaceId: UUID) throws { + private func ensureWorkspaceDirectoriesLocked(for workspaceId: UUID) throws -> Int { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) let existed = fileManager.fileExists(atPath: workspaceURL.path) - try fileManager.createDirectory(at: workspaceURL, withIntermediateDirectories: true) + try ensureDirectory(workspaceURL) let formatURL = workspaceURL.appendingPathComponent("format.json") + let version: Int if fileManager.fileExists(atPath: formatURL.path) { - try validateWorkspaceFormatLocked(workspaceId) + version = try validateWorkspaceFormatLocked(workspaceId) } else { if existed, !(try fileManager.contentsOfDirectory(at: workspaceURL, includingPropertiesForKeys: nil)).isEmpty { throw WorkspaceError.storageCorrupted("workspace store has no format version") } - try encoder.encode(StoreFormat(version: StoreFormat.currentVersion)).write(to: formatURL, options: .atomic) + try Self.durableWrite(encoder.encode(StoreFormat(version: StoreFormat.currentVersion)), to: formatURL) + version = StoreFormat.currentVersion } - try fileManager.createDirectory(at: checkpointsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) - try fileManager.createDirectory(at: snapshotsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) - try fileManager.createDirectory(at: blobsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) + let newObjects = !fileManager.fileExists(atPath: treesDirectoryURL(workspaceId: workspaceId).path) + try ensureDirectory(checkpointsDirectoryURL(workspaceId: workspaceId)) + try ensureDirectory(snapshotsDirectoryURL(workspaceId: workspaceId)) + try ensureDirectory(blobsDirectoryURL(workspaceId: workspaceId)) + try ensureDirectory(treesDirectoryURL(workspaceId: workspaceId)) + try ensureDirectory(packsDirectoryURL(workspaceId: workspaceId)) + if !existed || newObjects { + try Self.syncDirectory(workspaceURL) + try Self.syncDirectory(rootDirectory) + try Self.syncDirectory(rootDirectory.deletingLastPathComponent()) + } + return version } private func validateWorkspaceFormatIfPresent(_ workspaceId: UUID) async throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: workspaceURL.path) else { return } try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { - try validateWorkspaceFormatLocked(workspaceId) + _ = try validateWorkspaceFormatLocked(workspaceId) } } - private func validateWorkspaceFormatLocked(_ workspaceId: UUID) throws { + private func validateWorkspaceFormatLocked(_ workspaceId: UUID) throws -> Int { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) let formatURL = workspaceURL.appendingPathComponent("format.json") guard fileManager.fileExists(atPath: formatURL.path) else { throw WorkspaceError.storageCorrupted("workspace store has no format version") } let format = try decoder.decode(StoreFormat.self, from: Data(contentsOf: formatURL)) - guard format.version == StoreFormat.currentVersion else { + guard (1...StoreFormat.currentVersion).contains(format.version) else { throw WorkspaceError.storageCorrupted("unsupported store format version \(format.version)") } + return format.version } private func workspaceDirectoryURL(workspaceId: UUID) -> URL { @@ -1486,8 +1770,16 @@ actor FileCheckpointStore: CheckpointStore { rootDirectory.appendingPathComponent(".\(workspaceId.uuidString).format.lock", isDirectory: false) } - private func write(_ value: T, to url: URL) throws { - try encoder.encode(value).write(to: url, options: .atomic) + private enum WriteDurability { case atomic, publication } + + private func write( + _ value: T, to url: URL, durability: WriteDurability + ) throws { + let data = try encoder.encode(value) + switch durability { + case .publication: try Self.durableWrite(data, to: url, publication: true) + case .atomic: try data.write(to: url, options: .atomic) + } } private func read(_ type: T.Type, from url: URL) throws -> T { @@ -1499,12 +1791,485 @@ actor FileCheckpointStore: CheckpointStore { } private struct StoreFormat: Codable { - // Retained to read existing stores. Writers must preserve the sequence - // high-water mark; older writers are incompatible (see README). - static let currentVersion = 1 + // Existing v1 manifests and sequence sidecars remain readable. The v2 writer + // fence is published before new directory objects or packs (see README). + static let currentVersion = 2 var version: Int } + // One combined estimated-decoded budget per store, split between nodes and write catalog. + static let defaultCacheByteLimit = 32 * 1024 * 1024 + private var treeCache: CostLRU + private var packCatalog: CostLRU + private var packCatalogWorkspace: UUID? + private var packCatalogKey: String? + + private struct PackedBlob: Codable, Sendable, Equatable { + var pack: String + var offset: UInt64 + var size: UInt64 + + var estimatedCatalogBytes: Int { 256 + 2 * (64 + pack.utf8.count) } + } + + /// One writer, owned by one lifecycle-locked operation. Packs are capped at 16 MiB + /// (except a single large blob), with a 1 MiB write buffer. The first 32 unique + /// blobs stay loose so tiny checkpoints don't require pack/index overhead. + private final class BlobBatch { + let loose: URL + let packs: URL + var known: CostLRU + let fileManager: FileManager + let encoder: JSONEncoder + var seenLoose: Set = [] + var verifiedSizes: [URL: UInt64] = [:] + var verifiedPackSizes: [String: UInt64] = [:] + var insertedLines: [WorkspacePath: Int] = [:] + var summarizeInsertions = false + var wrotePacks = false + var count = 0 + var packID = UUID().uuidString + var handle: FileHandle? + var offset: UInt64 = 0 + var buffer = Data() + var index: [String: PackedBlob] = [:] + + init(loose: URL, packs: URL, known: CostLRU, fileManager: FileManager, encoder: JSONEncoder) { + self.loose = loose + self.packs = packs + self.known = known + self.fileManager = fileManager + self.encoder = encoder + } + + func store(_ data: Data, hash: String) throws -> PackedBlob? { + if let location = known.value(for: hash) { return location } + if seenLoose.contains(hash) { return nil } + let url = loose.appendingPathComponent(hash) + if fileManager.fileExists(atPath: url.path) { + seenLoose.insert(hash) + return nil + } + count += 1 + if count <= 32 { + try FileCheckpointStore.durableWrite(data, to: url) + seenLoose.insert(hash) + return nil + } + if offset > 0, offset + UInt64(data.count) > 16 * 1024 * 1024 { try finishPack() } + if handle == nil { + let url = packs.appendingPathComponent(packID + ".pack") + guard fileManager.createFile(atPath: url.path, contents: nil) else { + throw WorkspaceError.storageCorrupted("cannot create blob pack") + } + handle = try FileHandle(forWritingTo: url) + } + let location = PackedBlob(pack: packID, offset: offset, size: UInt64(data.count)) + if data.count >= 1024 * 1024 { + // Do not duplicate a large captured file in the pack buffer. + try flush() + try handle?.write(contentsOf: data) + } else { + buffer.append(data) + if buffer.count >= 1024 * 1024 { try flush() } + } + offset += UInt64(data.count) + index[hash] = location + known.insert(location, for: hash, cost: location.estimatedCatalogBytes) + return location + } + + func flush() throws { + if !buffer.isEmpty { + try handle?.write(contentsOf: buffer) + buffer.removeAll(keepingCapacity: true) + } + } + + func finishPack() throws { + guard let handle else { return } + try flush() + try handle.synchronize() + try handle.close() + self.handle = nil + try FileCheckpointStore.syncDirectory(packs) + try FileCheckpointStore.durableWrite(encoder.encode(index), to: packs.appendingPathComponent(packID + ".json")) + wrotePacks = true + packID = UUID().uuidString + offset = 0 + index.removeAll(keepingCapacity: true) + } + + func close() { try? handle?.close(); handle = nil } + deinit { close() } + } + + private func treesDirectoryURL(workspaceId: UUID) -> URL { + workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("trees", isDirectory: true) + } + + private func packsDirectoryURL(workspaceId: UUID) -> URL { + workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("packs", isDirectory: true) + } + + /// Constant-size invalidation avoids enumerating historical packs on every + /// checkpoint. A durable random generation also avoids directory-mtime ABA. + private func packGeneration(workspaceId: UUID) throws -> String? { + let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("packs.generation") + guard fileManager.fileExists(atPath: url.path) else { return nil } + let value = String(decoding: try Data(contentsOf: url), as: UTF8.self) + guard UUID(uuidString: value) != nil else { throw WorkspaceError.storageCorrupted("invalid pack catalog generation") } + return value + } + + private func advancePackGeneration(workspaceId: UUID) throws { + let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("packs.generation") + try Self.durableWrite(Data(UUID().uuidString.utf8), to: url) + } + + private func makeBlobBatch(workspaceId: UUID) throws -> BlobBatch { + let packs = packsDirectoryURL(workspaceId: workspaceId) + let key = try packGeneration(workspaceId: workspaceId) + var known = packCatalog + if packCatalogWorkspace != workspaceId || packCatalogKey != key { + known.removeAll() + for url in try artifactURLs(in: packs, pathExtension: "json") { + let entries = try read([String: PackedBlob].self, from: url) + for (hash, location) in entries { + guard Self.isValidContentHash(hash), UUID(uuidString: location.pack) != nil, + location.pack == url.deletingPathExtension().lastPathComponent, + fileManager.fileExists(atPath: packs.appendingPathComponent(location.pack + ".pack").path) + else { continue } // Interrupted, unreferenced packs are swept by compaction. + known.insert(location, for: hash, cost: location.estimatedCatalogBytes) + } + } + packCatalog = known + packCatalogWorkspace = workspaceId + packCatalogKey = key + } + let loose = blobsDirectoryURL(workspaceId: workspaceId) + return BlobBatch(loose: loose, packs: packs, known: known, fileManager: fileManager, encoder: compactEncoder) + } + + private func finish(_ batch: BlobBatch, workspaceId: UUID) throws { + try batch.finishPack() + if batch.wrotePacks { try advancePackGeneration(workspaceId: workspaceId) } + packCatalog = batch.known + packCatalogWorkspace = workspaceId + packCatalogKey = try packGeneration(workspaceId: workspaceId) + try Self.syncDirectory(treesDirectoryURL(workspaceId: workspaceId)) + } + + private func cacheTree(_ entries: [TreeEntry], hash: String, workspaceId: UUID) { + let key = workspaceId.uuidString + hash + let cost = 256 + key.utf8.count * 2 + entries.reduce(0) { $0 + $1.estimatedBytes } + treeCache.insert(entries, for: key, cost: cost) + } + + private func children(of node: SnapshotManifest.Node, workspaceId: UUID) throws -> [SnapshotManifest.Node] { + guard node.kind == .directory else { return [] } + guard let hash = node.treeHash else { return (node.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) } } + return try treeEntries(hash: hash, workspaceId: workspaceId).map { $0.node(in: node.path) } + } + + private func treeEntries(hash: String, workspaceId: UUID) throws -> [TreeEntry] { + guard Self.isValidContentHash(hash) else { throw WorkspaceError.storageCorrupted("invalid tree hash") } + let url = treesDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash) + guard fileManager.fileExists(atPath: url.path) else { throw WorkspaceError.storageCorrupted("missing tree \(hash)") } + let entries: [TreeEntry] + if let cached = treeCache.value(for: workspaceId.uuidString + hash) { + entries = cached + } else { + let bytes = try Data(contentsOf: url) + guard SHA256.hexDigest(of: bytes) == hash else { throw WorkspaceError.storageCorrupted("tree hash mismatch") } + entries = try decoder.decode([TreeEntry].self, from: bytes) + var previous: PathName? + for entry in entries { + try TreeEntry.validate(entry.name) + guard previous.map({ $0 < PathName(entry.name) }) ?? true else { + throw WorkspaceError.storageCorrupted("unsorted or duplicate tree child") + } + previous = PathName(entry.name) + } + cacheTree(entries, hash: hash, workspaceId: workspaceId) + } + return entries + } + + private func persistDirectory( + _ node: SnapshotManifest.Node, workspaceId: UUID, syncParent: Bool = false + ) throws -> SnapshotManifest.Node { + let entries = try (node.children ?? []).sorted { + PathName($0.path.name) < PathName($1.path.name) + }.map(TreeEntry.init) + let bytes = try compactEncoder.encode(entries) + let hash = SHA256.hexDigest(of: bytes) + let url = treesDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash) + if !fileManager.fileExists(atPath: url.path) { + try Self.durableWrite(bytes, to: url, syncParent: syncParent) + } + cacheTree(entries, hash: hash, workspaceId: workspaceId) + var reference = node + reference.children = nil + reference.treeHash = hash + return reference + } + + private func blobLocation(_ node: SnapshotManifest.Node, workspaceId: UUID) throws -> (url: URL, offset: UInt64, size: UInt64) { + guard let hash = node.contentHash, Self.isValidContentHash(hash) else { + throw WorkspaceError.storageCorrupted("invalid blob hash for \(node.path)") + } + if let packed = node.packed { + guard UUID(uuidString: packed.pack) != nil, packed.size == node.contentSize, + packed.offset <= UInt64.max - packed.size else { + throw WorkspaceError.storageCorrupted("invalid packed blob range") + } + return (packsDirectoryURL(workspaceId: workspaceId).appendingPathComponent(packed.pack + ".pack"), packed.offset, packed.size) + } + let url = blobURL(hash: hash, workspaceId: workspaceId) + return (url, 0, try node.contentSize ?? fileSize(at: url)) + } + + private func blobExists(_ node: SnapshotManifest.Node, workspaceId: UUID, batch: BlobBatch) -> Bool { + if let packed = node.packed, let size = batch.verifiedPackSizes[packed.pack] { + return packed.offset <= size && packed.size <= size - packed.offset + } + guard let location = try? blobLocation(node, workspaceId: workspaceId) else { return false } + if let size = batch.verifiedSizes[location.url] { return location.offset <= size && location.size <= size - location.offset } + guard let size = try? fileSize(at: location.url) else { return false } + batch.verifiedSizes[location.url] = size + if let packed = node.packed { batch.verifiedPackSizes[packed.pack] = size } + return location.offset <= size && location.size <= size - location.offset + } + + /// Materialization walks most files in a pack together. Map at most four packs + /// per load, then copy only each selected range into its Snapshot.File. No mapped + /// pack escapes the lifecycle-locked load or pins deleted packs after compaction. + private final class SnapshotBlobCache { + var packs: [String: Data] = [:] + var order: [String] = [] + } + + private func materializedBlob(_ node: SnapshotManifest.Node, workspaceId: UUID, cache: SnapshotBlobCache) throws -> Data { + let bytes: Data + let offset: UInt64 + let size: UInt64 + if let packed = node.packed { + guard packed.size == node.contentSize else { throw WorkspaceError.storageCorrupted("invalid packed blob size") } + if let cached = cache.packs[packed.pack] { + bytes = cached + } else { + let location = try blobLocation(node, workspaceId: workspaceId) + do { bytes = try Data(contentsOf: location.url, options: .mappedIfSafe) } + catch { + if Snapshot.isMissingEntry(error) { throw WorkspaceError.storageCorrupted("missing content blob for \(node.path)") } + throw error + } + if cache.order.count == 4 { cache.packs.removeValue(forKey: cache.order.removeFirst()) } + cache.packs[packed.pack] = bytes + cache.order.append(packed.pack) + } + offset = packed.offset + size = packed.size + } else { + let location = try blobLocation(node, workspaceId: workspaceId) + do { bytes = try Data(contentsOf: location.url) } + catch { + if Snapshot.isMissingEntry(error) { throw WorkspaceError.storageCorrupted("missing content blob for \(node.path)") } + throw error + } + offset = 0 + size = location.size + if size == UInt64(bytes.count) { return bytes } + } + guard offset <= UInt64(bytes.count), size <= UInt64(bytes.count) - offset else { + throw WorkspaceError.storageCorrupted("truncated content blob for \(node.path)") + } + return bytes.subdata(in: Int(offset).. Data { + let location = try blobLocation(node, workspaceId: workspaceId) + guard fileManager.fileExists(atPath: location.url.path) else { + throw WorkspaceError.storageCorrupted("missing content blob for \(node.path)") + } + let handle = try FileHandle(forReadingFrom: location.url) + defer { try? handle.close() } + let physicalSize = try handle.seekToEnd() + guard location.offset <= physicalSize, location.size <= physicalSize - location.offset else { + throw WorkspaceError.storageCorrupted("truncated content blob for \(node.path)") + } + guard offset < location.size else { return Data() } + try handle.seek(toOffset: location.offset + offset) + let count = min(UInt64(length ?? Int.max), location.size - offset) + let bytes = try handle.read(upToCount: Int(count)) ?? Data() + guard bytes.count == Int(count) else { throw WorkspaceError.storageCorrupted("short blob read") } + return bytes + } + + func resolveSnapshotFile(id: UUID, workspaceId: UUID, path: WorkspacePath) async throws -> (path: WorkspacePath, size: UInt64) { + try await validateWorkspaceFormatIfPresent(workspaceId) + guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { + throw WorkspaceError.revisionDataNotFound(id) + } + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { + throw WorkspaceError.revisionDataNotFound(id) + } + return try RevisionIndex.resolveReadableFile(at: path) { current in + guard let node = try manifestNode(at: current, in: manifest.root, workspaceId: workspaceId) else { return nil } + if node.kind == .file { + return .file(path: current, size: try blobLocation(node, workspaceId: workspaceId).size, + permissions: node.permissions ?? .defaultFile, contentID: node.contentHash ?? "") + } + return try revisionEntry(from: node, workspaceId: workspaceId, depth: 0) + } + } + } + + /// Shallow metadata: a list decodes its directory, never grandchildren. + func snapshotEntry(id: UUID, workspaceId: UUID, path: WorkspacePath, includeChildren: Bool) async throws -> RevisionIndex.Entry? { + try await validateWorkspaceFormatIfPresent(workspaceId) + guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { + throw WorkspaceError.revisionDataNotFound(id) + } + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { + throw WorkspaceError.revisionDataNotFound(id) + } + guard let node = try manifestNode(at: path, in: manifest.root, workspaceId: workspaceId) else { return nil } + return try revisionEntry(from: node, workspaceId: workspaceId, depth: includeChildren ? 1 : 0) + } + } + + private func creationSummary(_ root: SnapshotManifest.Node, workspaceId: UUID, insertedLines: [WorkspacePath: Int]) throws -> ChangeSet.Summary { + var paths: [WorkspacePath] = [] + var files = 0 + func visit(_ node: SnapshotManifest.Node) throws { + if node.kind == .missing { return } + paths.append(node.path) + if node.kind == .file { files += 1 } + for child in try children(of: node, workspaceId: workspaceId) { try visit(child) } + } + try visit(root) + let additions = insertedLines.values.reduce(0, +) + return .init(changedPathCount: paths.count, touchedPaths: paths, hasTextChanges: additions > 0, + statistics: .init(changedPathCount: paths.count, changedFileCount: files, additions: additions, deletions: 0)) + } + + private func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID) async throws -> ChangeSet { + if old == new { return ChangeSet(changes: []) } + if old.kind == .directory, new.kind == .directory { + var changes: [FileChange] = (old.permissions ?? .defaultDirectory) == (new.permissions ?? .defaultDirectory) ? [] : [.init(path: new.path, kind: .directory, effect: .modified)] + var omissions: [TextDiffOmission] = [] + if old.treeHash != nil, old.treeHash == new.treeHash { return ChangeSet(changes: changes) } + let before = Dictionary(uniqueKeysWithValues: try children(of: old, workspaceId: workspaceId).map { (PathName($0.path.name), $0) }) + let after = Dictionary(uniqueKeysWithValues: try children(of: new, workspaceId: workspaceId).map { (PathName($0.path.name), $0) }) + for key in Set(before.keys).union(after.keys).sorted() { + let name = key.string + let result = try await compareNodes( + before[key] ?? .init(kind: .missing, path: old.path.appending(name)), + after[key] ?? .init(kind: .missing, path: new.path.appending(name)), workspaceId: workspaceId) + changes += result.changes + omissions += result.textDiffOmissions + } + return ChangeSet(changes: changes, textDiffOmissions: omissions) + } + // Newly created/deleted trees need their complete summaries; normal incremental + // changes reach this path only for changed leaves. + let before = RevisionIndex(id: UUID(), root: old.path, entry: try revisionEntry(from: old, workspaceId: workspaceId)) + let after = RevisionIndex(id: UUID(), root: new.path, entry: try revisionEntry(from: new, workspaceId: workspaceId)) + return try await ChangeSet.compare(before: before, after: after, maxTextBytes: 1_000_000, + loadBefore: { path in try await self.readNodeFile(path, root: old, workspaceId: workspaceId) }, + loadAfter: { path in try await self.readNodeFile(path, root: new, workspaceId: workspaceId) }) + } + + private func readNodeFile(_ path: WorkspacePath, root: SnapshotManifest.Node, workspaceId: UUID) throws -> Data { + guard let node = try manifestNode(at: path, in: root, workspaceId: workspaceId) else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) + } + return try readBlob(node, workspaceId: workspaceId) + } + + private func requireCurrentFormatUnlocked(workspaceId: UUID) throws { + let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("format.json") + guard try read(StoreFormat.self, from: url).version == StoreFormat.currentVersion else { + throw WorkspaceError.storageCorrupted("store format changed after preparation") + } + } + + /// v1 manifests remain readable. Publish the writer fence before any v2 objects. + /// Migration runs under format -> lifecycle -> mutation locks, in that order. + private func upgradeFormatUnlocked(workspaceId: UUID) throws { + let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("format.json") + let format = try read(StoreFormat.self, from: url) + guard (1...StoreFormat.currentVersion).contains(format.version) else { + throw WorkspaceError.storageCorrupted("unsupported store format version \(format.version)") + } + if format.version < StoreFormat.currentVersion { + try Self.durableWrite(encoder.encode(StoreFormat(version: StoreFormat.currentVersion)), to: url) + } + } + + private nonisolated static func durableWrite( + _ data: Data, to url: URL, syncParent: Bool = true, publication: Bool = false + ) throws { +#if canImport(Darwin) || canImport(Glibc) + let directory = url.deletingLastPathComponent() + let temporary = directory.appendingPathComponent(".pending-" + UUID().uuidString + "-" + url.lastPathComponent) + let fd = open(temporary.path, O_WRONLY | O_CREAT | O_EXCL, 0o644) + guard fd >= 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + defer { close(fd); _ = unlink(temporary.path) } + try data.withUnsafeBytes { bytes in + var written = 0 + while written < bytes.count { +#if canImport(Darwin) + let count = Darwin.write(fd, bytes.baseAddress!.advanced(by: written), bytes.count - written) +#else + let count = Glibc.write(fd, bytes.baseAddress!.advanced(by: written), bytes.count - written) +#endif + if count < 0, errno == EINTR { continue } + guard count > 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + written += count + } + } + while fsync(fd) != 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard rename(temporary.path, url.path) == 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + if syncParent { try syncDirectory(directory) } +#if canImport(Darwin) + if publication { + // Flush the drive cache after both file and directory metadata are synced. + while fcntl(fd, F_FULLFSYNC) != 0 { + if errno == EINTR { continue } + // Some network/FUSE volumes do not support this request. Keep + // the file and directory fsync guarantees already established. + break + } + } +#endif +#else + try data.write(to: url, options: .atomic) + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.synchronize() +#endif + } + + private nonisolated static func syncDirectory(_ url: URL) throws { +#if canImport(Darwin) || canImport(Glibc) + let fd = open(url.path, O_RDONLY) + guard fd >= 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + defer { close(fd) } + while fsync(fd) != 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } +#endif + } + private func acquireGate(for path: String) async { let path = PathName(path) if heldLocks.insert(path).inserted { return } diff --git a/Sources/Workspace/RevisionIndex.swift b/Sources/Workspace/RevisionIndex.swift index 8dff252..bba08b3 100644 --- a/Sources/Workspace/RevisionIndex.swift +++ b/Sources/Workspace/RevisionIndex.swift @@ -77,7 +77,7 @@ struct RevisionIndex: Sendable, Equatable { path: value.path, size: UInt64(value.data.count), permissions: value.permissions, - contentID: SHA256.hexDigest(of: value.data) + contentID: value.contentID ?? SHA256.hexDigest(of: value.data) ) case let .directory(value): .directory(path: value.path, permissions: value.permissions, children: value.children.map(index)) @@ -92,13 +92,20 @@ struct RevisionIndex: Sendable, Equatable { /// Resolve each component in the revision namespace, including directory symlinks. func readableFile(at path: WorkspacePath) throws -> (path: WorkspacePath, size: UInt64) { + try Self.resolveReadableFile(at: path) { entry(at: $0) } + } + + /// Shared by tree-backed and materialized revisions; keep symlink semantics in one place. + static func resolveReadableFile( + at path: WorkspacePath, lookup: (WorkspacePath) throws -> Entry? + ) throws -> (path: WorkspacePath, size: UInt64) { var pending = path.components var current = WorkspacePath.root var hops = 0 while !pending.isEmpty { let component = pending.removeFirst() current = current.appending(component) - guard let entry = entry(at: current) else { + guard let entry = try lookup(current) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } switch entry { @@ -121,11 +128,15 @@ struct RevisionIndex: Sendable, Equatable { } private static func entry(at path: WorkspacePath, in entry: Entry) -> Entry? { - if entry.path == path { return entry } - guard case let .directory(_, _, children) = entry else { return nil } - for child in children where path.isWithin(child.path) { - return self.entry(at: path, in: child) + let components = path.components + guard path.isWithin(entry.path) else { return nil } + let prefix = entry.path.components + var current = entry + for component in components.dropFirst(prefix.count) { + guard case let .directory(_, _, children) = current, + let child = children.first(where: { PathName($0.path.name) == PathName(component) }) else { return nil } + current = child } - return nil + return current } } diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index f1719e4..4728c9d 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -45,14 +45,24 @@ struct Snapshot: Sendable, Codable, Equatable { /// The captured path. var path: WorkspacePath /// The file's contents at the time of capture. - var data: Data + var data: Data { didSet { contentID = nil } } /// The file's POSIX permissions at the time of capture. var permissions: POSIXPermissions + // Captured once with the bytes; excluded from value equality and persistence. + var contentID: String? = nil + + private enum CodingKeys: String, CodingKey { case path, data, permissions } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.path == rhs.path && lhs.permissions == rhs.permissions && lhs.data == rhs.data + } + /// Creates a file entry. - init(path: WorkspacePath, data: Data, permissions: POSIXPermissions) { + init(path: WorkspacePath, data: Data, permissions: POSIXPermissions, contentID: String? = nil) { self.path = path self.data = data + self.contentID = contentID self.permissions = permissions } } diff --git a/Sources/Workspace/Support/CostLRU.swift b/Sources/Workspace/Support/CostLRU.swift new file mode 100644 index 0000000..f81cf51 --- /dev/null +++ b/Sources/Workspace/Support/CostLRU.swift @@ -0,0 +1,59 @@ +/// A byte-budgeted LRU. Dictionary links make lookup, promotion and eviction O(1). +/// Costs include estimated decoded storage and bookkeeping, not encoded JSON sizes. +struct CostLRU: Sendable { + private struct Entry: Sendable { + var value: Value + var cost: Int + var previous: Key? + var next: Key? + } + + let limit: Int + private var entries: [Key: Entry] = [:] + private var oldest: Key? + private var newest: Key? + private(set) var cost = 0 + var count: Int { entries.count } + + init(limit: Int) { self.limit = max(0, limit) } + + mutating func value(for key: Key) -> Value? { + guard let entry = entries[key] else { return nil } + promote(key) + return entry.value + } + + mutating func insert(_ value: Value, for key: Key, cost entryCost: Int) { + remove(key) + guard entryCost >= 0, entryCost <= limit else { return } + while cost > limit - entryCost, let key = oldest { remove(key) } + entries[key] = Entry(value: value, cost: entryCost, previous: newest) + if let newest { entries[newest]?.next = key } else { oldest = key } + newest = key + cost += entryCost + } + + private mutating func promote(_ key: Key) { + guard newest != key, let entry = entries[key] else { return } + if let previous = entry.previous { entries[previous]?.next = entry.next } else { oldest = entry.next } + if let next = entry.next { entries[next]?.previous = entry.previous } + entries[key]?.previous = newest + entries[key]?.next = nil + if let newest { entries[newest]?.next = key } + newest = key + } + + private mutating func remove(_ key: Key) { + guard let entry = entries.removeValue(forKey: key) else { return } + if let previous = entry.previous { entries[previous]?.next = entry.next } else { oldest = entry.next } + if let next = entry.next { entries[next]?.previous = entry.previous } else { newest = entry.previous } + cost -= entry.cost + } + + mutating func removeAll() { + entries.removeAll() + oldest = nil + newest = nil + cost = 0 + } +} diff --git a/Sources/Workspace/Support/SHA256.swift b/Sources/Workspace/Support/SHA256.swift index 7c26550..d35488b 100644 --- a/Sources/Workspace/Support/SHA256.swift +++ b/Sources/Workspace/Support/SHA256.swift @@ -1,77 +1,93 @@ import Foundation +#if canImport(CryptoKit) +import CryptoKit +#endif -/// A minimal, dependency-free SHA-256 implementation (FIPS 180-4). +/// SHA-256 via system CryptoKit on Apple platforms, with a dependency-free FIPS 180-4 fallback. /// /// Used to content-address snapshot blobs in ``FileCheckpointStore``. Kept internal: it is not a /// general-purpose cryptography API. enum SHA256 { /// Returns the lowercase hex digest of `data`. static func hexDigest(of data: Data) -> String { - digest(of: data).map { String(format: "%02x", $0) }.joined() + let digits = Array("0123456789abcdef".utf8) + var hex = [UInt8](repeating: 0, count: 64) + for (index, byte) in digest(of: data).enumerated() { + hex[index * 2] = digits[Int(byte >> 4)] + hex[index * 2 + 1] = digits[Int(byte & 15)] + } + return String(decoding: hex, as: UTF8.self) } /// Returns the 32-byte SHA-256 digest of `data`. static func digest(of data: Data) -> [UInt8] { +#if canImport(CryptoKit) + return Array(CryptoKit.SHA256.hash(data: data)) +#else var h: [UInt32] = [ 0x6a09_e667, 0xbb67_ae85, 0x3c6e_f372, 0xa54f_f53a, 0x510e_527f, 0x9b05_688c, 0x1f83_d9ab, 0x5be0_cd19, ] - // Padding: append 0x80, zeros, then the 64-bit big-endian bit length. - var message = [UInt8](data) - let bitLength = UInt64(message.count) * 8 - message.append(0x80) - while message.count % 64 != 56 { - message.append(0) - } - for shift in stride(from: 56, through: 0, by: -8) { - message.append(UInt8(truncatingIfNeeded: bitLength >> UInt64(shift))) - } - + // Process the input in place. Only the final (at most two) blocks need padding. var w = [UInt32](repeating: 0, count: 64) - for blockStart in stride(from: 0, to: message.count, by: 64) { - for index in 0..<16 { - let offset = blockStart + index * 4 - w[index] = UInt32(message[offset]) << 24 - | UInt32(message[offset + 1]) << 16 - | UInt32(message[offset + 2]) << 8 - | UInt32(message[offset + 3]) - } - for index in 16..<64 { - let s0 = rotr(w[index - 15], 7) ^ rotr(w[index - 15], 18) ^ (w[index - 15] >> 3) - let s1 = rotr(w[index - 2], 17) ^ rotr(w[index - 2], 19) ^ (w[index - 2] >> 10) - w[index] = w[index - 16] &+ s0 &+ w[index - 7] &+ s1 + data.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + let fullBytes = bytes.count / 64 * 64 + var tail = Array(bytes[fullBytes...]) + tail.append(0x80) + while tail.count % 64 != 56 { tail.append(0) } + let bitLength = UInt64(bytes.count) * 8 + for shift in stride(from: 56, through: 0, by: -8) { + tail.append(UInt8(truncatingIfNeeded: bitLength >> UInt64(shift))) } + for blockStart in stride(from: 0, to: fullBytes + tail.count, by: 64) { + for index in 0..<16 { + let offset = blockStart + index * 4 + if blockStart < fullBytes { + w[index] = UInt32(bigEndian: bytes.loadUnaligned(fromByteOffset: offset, as: UInt32.self)) + } else { + let offset = offset - fullBytes + w[index] = UInt32(tail[offset]) << 24 | UInt32(tail[offset + 1]) << 16 + | UInt32(tail[offset + 2]) << 8 | UInt32(tail[offset + 3]) + } + } + for index in 16..<64 { + let s0 = rotr(w[index - 15], 7) ^ rotr(w[index - 15], 18) ^ (w[index - 15] >> 3) + let s1 = rotr(w[index - 2], 17) ^ rotr(w[index - 2], 19) ^ (w[index - 2] >> 10) + w[index] = w[index - 16] &+ s0 &+ w[index - 7] &+ s1 + } + + var a = h[0], b = h[1], c = h[2], d = h[3] + var e = h[4], f = h[5], g = h[6], hh = h[7] - var a = h[0], b = h[1], c = h[2], d = h[3] - var e = h[4], f = h[5], g = h[6], hh = h[7] + for index in 0..<64 { + let s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25) + let ch = (e & f) ^ (~e & g) + let temp1 = hh &+ s1 &+ ch &+ Self.k[index] &+ w[index] + let s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22) + let maj = (a & b) ^ (a & c) ^ (b & c) + let temp2 = s0 &+ maj - for index in 0..<64 { - let s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25) - let ch = (e & f) ^ (~e & g) - let temp1 = hh &+ s1 &+ ch &+ Self.k[index] &+ w[index] - let s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22) - let maj = (a & b) ^ (a & c) ^ (b & c) - let temp2 = s0 &+ maj + hh = g + g = f + f = e + e = d &+ temp1 + d = c + c = b + b = a + a = temp1 &+ temp2 + } - hh = g - g = f - f = e - e = d &+ temp1 - d = c - c = b - b = a - a = temp1 &+ temp2 + h[0] &+= a + h[1] &+= b + h[2] &+= c + h[3] &+= d + h[4] &+= e + h[5] &+= f + h[6] &+= g + h[7] &+= hh } - h[0] &+= a - h[1] &+= b - h[2] &+= c - h[3] &+= d - h[4] &+= e - h[5] &+= f - h[6] &+= g - h[7] &+= hh } var out: [UInt8] = [] @@ -83,6 +99,7 @@ enum SHA256 { out.append(UInt8(truncatingIfNeeded: word)) } return out +#endif } private static func rotr(_ value: UInt32, _ amount: UInt32) -> UInt32 { diff --git a/Sources/Workspace/TextDiff.swift b/Sources/Workspace/TextDiff.swift index 17f61f1..b0e8106 100644 --- a/Sources/Workspace/TextDiff.swift +++ b/Sources/Workspace/TextDiff.swift @@ -226,14 +226,17 @@ private func diffLineTokens(in text: String) -> [DiffLineToken] { var tokens: [DiffLineToken] = [] // CRLF is one Swift Character. Split UTF-8 code units so LF alone is the delimiter, // preserving CR in the line's content and distinguishing a terminated final line. - let bytes = text.utf8 - var start = bytes.startIndex - for index in bytes.indices where bytes[index] == 0x0A { - tokens.append(DiffLineToken(text: String(decoding: bytes[start.. Data + } + + private func checkpointAccess(_ id: UUID) async throws -> CheckpointAccess { + try await ensureLoaded() + try await reconcileCheckpointsWithStore() + let checkpoint = try checkpointOrThrow(id: id) + let store = store + return CheckpointAccess(checkpoint: checkpoint, readRange: { path, offset, length in + guard let data = try await store.readSnapshotFile( + id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId, + path: path, offset: offset, length: length + ) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } + return data + }) + } + + private func checkpointEntry(_ id: UUID, at path: WorkspacePath, includeChildren: Bool) async throws -> RevisionIndex.Entry? { + let access = try await checkpointAccess(id) + return try await store.snapshotEntry( + id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceId, + path: path, includeChildren: includeChildren + ) + } + public func info(_ path: WorkspacePath, at revision: Revision = .current) async throws -> FileInfo { switch revision { case .current: return try await filesystem.stat(path: path) - case .checkpoint: - let resolved = try await resolvedRevision(revision) - guard let entry = resolved.index.entry(at: path) else { + case let .checkpoint(id): + guard let entry = try await checkpointEntry(id, at: path, includeChildren: false) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } return Self.info(for: entry) @@ -171,9 +177,8 @@ extension Workspace { switch revision { case .current: return try await filesystem.listDirectory(path: path) - case .checkpoint: - let resolved = try await resolvedRevision(revision) - guard let entry = resolved.index.entry(at: path), + case let .checkpoint(id): + guard let entry = try await checkpointEntry(id, at: path, includeChildren: true), case let .directory(_, _, children) = entry else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) diff --git a/Tests/WorkspaceTests/CheckpointFormatTests.swift b/Tests/WorkspaceTests/CheckpointFormatTests.swift new file mode 100644 index 0000000..21f34f6 --- /dev/null +++ b/Tests/WorkspaceTests/CheckpointFormatTests.swift @@ -0,0 +1,577 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Checkpoint tree format") +struct CheckpointFormatTests { + // This fixture was emitted by the original v1 FileCheckpointStore, including + // its mutation log and sequence sidecar. It is deliberately not re-encoded here. + @Test func `v1 fixture remains readable compactable and appendable with a v2 writer fence`() async throws { + let directory = try TestSupport.temporaryDirectory("LegacyStore") + defer { TestSupport.remove(directory) } + let source = try #require(Bundle.module.url(forResource: "v1", withExtension: nil, subdirectory: "Fixtures")) + let root = directory.appendingPathComponent("store") + try FileManager.default.copyItem(at: source, to: root) + let id = UUID(uuidString: "10000000-0000-0000-0000-000000000001")! + let store = FileCheckpointStore(rootDirectory: root) + let old = try await store.listCheckpoints(workspaceId: id) + #expect(old.count == 2) + let legacyFile = try await store.resolveSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: "/link") + #expect(legacyFile.path == "/nested/deep/file") + #expect(try await store.readSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: legacyFile.path, offset: 0, length: nil) == Data("nested legacy\n".utf8)) + + #expect(try await store.readSnapshotFile(id: old[0].snapshotId, workspaceId: id, path: "/note", offset: 0, length: nil) == Data("legacy checkpoint\n".utf8)) + #expect(try await store.readSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: "/note", offset: 0, length: nil) == Data("legacy updated\n".utf8)) + let dry = try await store.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: true) + let compacted = try await store.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: false) + #expect(dry.report.after == compacted.report.after) + #expect(compacted.checkpoints.first?.parentID == nil) + // Legacy nil directory permissions mean defaultDirectory, not a modification. + let legacyURL = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(old[1].snapshotId).json") + var legacyJSON = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: legacyURL)) as? [String: Any]) + var legacyRoot = try #require(legacyJSON["root"] as? [String: Any]) + legacyRoot.removeValue(forKey: "permissions") + legacyJSON["root"] = legacyRoot + try JSONSerialization.data(withJSONObject: legacyJSON).write(to: legacyURL) + let restored = InMemoryFileSystem() + let legacySnapshot = try #require(await store.loadSnapshot(id: old[1].snapshotId, workspaceId: id)) + try await Snapshot.restore(legacySnapshot, to: restored) + let unchanged = try await store.captureRevision(from: restored, draft: draft(id, parent: old[1].id)) + #expect(unchanged.summary.changedPathCount == 0) + + let next = try await store.saveRevision(snapshot(count: 4), draft: draft(id, parent: old[1].id)) + #expect(next.mutationCursor == 2) + let mutation = try await store.appendMutation(Mutation(sequence: 0, workspaceID: id, operation: .edit, changes: ChangeSet())) + #expect(mutation.sequence == 3) + let reopened = FileCheckpointStore(rootDirectory: root) + #expect(try await reopened.readSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: "/note", offset: 0, length: nil) == Data("legacy updated\n".utf8)) + #expect(try await reopened.loadSnapshot(id: next.snapshotId, workspaceId: id)?.entry == snapshot(count: 4).entry) + let format = root.appendingPathComponent(id.uuidString).appendingPathComponent("format.json") + // The v1 writer's actual admission rule: refuse any version other than 1, + // before creating or changing checkpoint/mutation artifacts. + struct V1Format: Decodable { var version: Int } + func v1WriterAdmission() throws { + let value = try JSONDecoder().decode(V1Format.self, from: Data(contentsOf: format)) + guard value.version == 1 else { throw WorkspaceError.storageCorrupted("unsupported store format version \(value.version)") } + } + #expect(throws: WorkspaceError.self) { try v1WriterAdmission() } + let newManifest = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(next.snapshotId).json") + #expect(throws: DecodingError.self) { + _ = try JSONDecoder().decode(V1SnapshotManifest.self, from: Data(contentsOf: newManifest)) + } + // The copied legacy decoder also succeeds on the real legacy fixture. + let oldManifest = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(old[1].snapshotId).json") + #expect(try JSONDecoder().decode(V1SnapshotManifest.self, from: Data(contentsOf: oldManifest)).version == 2) + + #expect(try await reopened.listMutations(workspaceId: id).map(\.sequence) == [1, 2, 3]) + } + + @Test func `one changed leaf writes only its ancestor trees and reopens correctly`() async throws { + let root = try TestSupport.temporaryDirectory("TreeReuse") + let files = try TestSupport.temporaryDirectory("TreeReuseFiles") + defer { TestSupport.remove(root); TestSupport.remove(files) } + let fs = try LocalFileSystem(root: files) + for directory in 0..<12 { + let path = WorkspacePath(normalizing: "/dir\(directory)/deep/leaf") + try await fs.createDirectory(path: path.parent, recursive: true) + try await fs.writeFile(path: path, data: Data("old \(directory)\n".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: Date(timeIntervalSince1970: 1)], ofItemAtPath: files.appendingPathComponent(String(path.string.dropFirst())).path) + } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + let first = try await store.captureRevision(from: fs, draft: draft(id)) + let trees = root.appendingPathComponent(id.uuidString).appendingPathComponent("trees") + let original = try artifacts(trees) + try await fs.writeFile(path: "/dir0/deep/leaf", data: Data("new\n".utf8), append: false) + let second = try await store.captureRevision(from: fs, draft: draft(id, parent: first.id)) + let updated = try artifacts(trees) + #expect(updated.count - original.count == 3) // root, dir0, deep + for (name, bytes) in original { #expect(updated[name] == bytes) } + #expect(second.summary.touchedPaths == ["/dir0/deep/leaf"]) + let reader = FileCheckpointStore(rootDirectory: root) + #expect(try await reader.readSnapshotFile(id: first.snapshotId, workspaceId: id, path: "/dir0/deep/leaf", offset: 0, length: nil) == Data("old 0\n".utf8)) + #expect(try await reader.readSnapshotFile(id: second.snapshotId, workspaceId: id, path: "/dir0/deep/leaf", offset: 0, length: nil) == Data("new\n".utf8)) + let third = try await store.captureRevision(from: fs, draft: draft(id, parent: second.id)) + #expect(third.summary.changedPathCount == 0) + #expect(try artifacts(trees) == updated) + // A cold range read must not decode an unrelated subtree. Corruption there + // is still reported by operations that need the complete snapshot. + let manifestURL = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(third.snapshotId.uuidString).json") + let manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: manifestURL)) as? [String: Any]) + let rootNode = try #require(manifest["treeRoot"] as? [String: Any]) + let rootHash = try #require(rootNode["treeHash"] as? String) + let children = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: trees.appendingPathComponent(rootHash))) as? [[String: Any]]) + let unrelated = try #require(children.first { ($0["name"] as? String) == "dir11" }) + let unrelatedHash = try #require(unrelated["treeHash"] as? String) + try FileManager.default.removeItem(at: trees.appendingPathComponent(unrelatedHash)) + let cold = FileCheckpointStore(rootDirectory: root) + #expect(try await cold.resolveSnapshotFile(id: third.snapshotId, workspaceId: id, path: "/dir0/deep/leaf").size == 4) + #expect(try await cold.readSnapshotFile(id: third.snapshotId, workspaceId: id, path: "/dir0/deep/leaf", offset: 1, length: 2) == Data("ew".utf8)) + let workspace = Workspace(workspaceID: id, fileSystem: fs, persistence: .directory(root)) + #expect(try await workspace.info("/dir0/deep/leaf", at: .checkpoint(third.id)).size == 4) + #expect(try await workspace.list("/dir0", at: .checkpoint(third.id)).map(\.name) == ["deep"]) + #expect(try await workspace.list(.root, at: .checkpoint(third.id)).count == 12) + await #expect(throws: WorkspaceError.self) { _ = try await cold.loadSnapshot(id: third.snapshotId, workspaceId: id) } + } + + @Test func `materialized saves reuse trees without trusting uncaptured timestamps`() async throws { + let root = try TestSupport.temporaryDirectory("SavedTreeReuse") + let files = try TestSupport.temporaryDirectory("SavedTreeFiles") + defer { TestSupport.remove(root); TestSupport.remove(files) } + let fs = try LocalFileSystem(root: files) + try await fs.createDirectory(path: "/a/b", recursive: true) + try await fs.writeFile(path: "/a/b/note", data: Data("old".utf8), append: false) + let fileURL = files.appendingPathComponent("a/b/note") + let oldDate = Date(timeIntervalSince1970: 1) + try FileManager.default.setAttributes([.modificationDate: oldDate], ofItemAtPath: fileURL.path) + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + let first = try await store.captureRevision(from: fs, draft: draft(id)) + var materialized = try #require(await store.loadSnapshot(id: first.snapshotId, workspaceId: id)) + materialized.id = UUID() + let trees = root.appendingPathComponent(id.uuidString).appendingPathComponent("trees") + let before = try artifacts(trees) + let saved = try await store.saveRevision(materialized, draft: draft(id, parent: first.id)) + #expect(try artifacts(trees) == before) + try await fs.writeFile(path: "/a/b/note", data: Data("new".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: oldDate], ofItemAtPath: fileURL.path) + let recaptured = try await store.captureRevision(from: fs, draft: draft(id, parent: saved.id)) + #expect(recaptured.summary.touchedPaths == ["/a/b/note"]) + #expect(try await store.readSnapshotFile(id: recaptured.snapshotId, workspaceId: id, path: "/a/b/note", offset: 0, length: nil) == Data("new".utf8)) + } + + @Test func `pack compaction reclaims dead entries and dry run matches physical usage`() async throws { + let root = try TestSupport.temporaryDirectory("PackCompaction") + defer { TestSupport.remove(root) } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + let first = try await store.saveRevision(snapshot(count: 100), draft: draft(id)) + let otherWriter = FileCheckpointStore(rootDirectory: root) + _ = try await otherWriter.saveRevision(snapshot(count: 100), draft: draft(id, parent: first.id)) + let kept = snapshot(count: 40, start: 60) + let second = try await store.saveRevision(kept, draft: draft(id, parent: first.id)) + let workspace = root.appendingPathComponent(id.uuidString) + let packs = workspace.appendingPathComponent("packs") + #expect(try FileManager.default.contentsOfDirectory(atPath: packs.path).contains { $0.hasSuffix(".pack") }) + let oldDirectoryDate = try #require(packs.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate) + // An interrupted pack write with no index must also be swept. + let orphan = packs.appendingPathComponent(".pending-" + UUID().uuidString + ".pack") + try Data("unpublished".utf8).write(to: orphan) + let before = try artifacts(packs) + let dry = try await store.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: true) + #expect(try artifacts(packs) == before) + #expect(dry.report.after.blobCount == 40) + #expect(dry.report.after.blobBytes == (60..<100).reduce(UInt64(0)) { $0 + UInt64(content($1).count) }) + let applied = try await store.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: false) + #expect(applied.report.after == dry.report.after) + #expect(try await store.storageStatistics(workspaceId: id) == dry.report.after) + let reader = FileCheckpointStore(rootDirectory: root) + #expect(try await reader.loadSnapshot(id: second.snapshotId, workspaceId: id)?.entry == kept.entry) + #expect(!FileManager.default.fileExists(atPath: orphan.path)) + #expect(try await reader.compact(workspaceId: id, retaining: .all, dryRun: false).report.reclaimedBytes == 0) + // This instance cached the old pack catalog before compaction. It must + // refresh before deduplicating a new root against those hashes. + try FileManager.default.setAttributes([.modificationDate: oldDirectoryDate], ofItemAtPath: packs.path) + let appended = try await otherWriter.saveRevision(snapshot(count: 100), draft: draft(id)) + #expect(try await reader.loadSnapshot(id: appended.snapshotId, workspaceId: id)?.entry == snapshot(count: 100).entry) + } + + @Test func `compaction resumes after only some roots moved to replacement packs`() async throws { + let root = try TestSupport.temporaryDirectory("InterruptedRepack") + defer { TestSupport.remove(root) } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + let first = try await store.saveRevision(snapshot(count: 100), draft: draft(id)) + let second = try await store.saveRevision(snapshot(count: 100), draft: draft(id, parent: first.id)) + let directory = root.appendingPathComponent(id.uuidString) + let packs = directory.appendingPathComponent("packs") + let oldPack = try #require(FileManager.default.contentsOfDirectory(at: packs, includingPropertiesForKeys: nil).first { $0.pathExtension == "pack" }) + let newID = UUID().uuidString + try FileManager.default.copyItem(at: oldPack, to: packs.appendingPathComponent(newID + ".pack")) + let unindexed = try FileHandle(forWritingTo: packs.appendingPathComponent(newID + ".pack")) + try unindexed.seekToEnd() + try unindexed.write(contentsOf: Data("unindexed tail".utf8)) + try unindexed.close() + // Model the durable state after repacking and updating just one snapshot root. + let oldID = oldPack.deletingPathExtension().lastPathComponent + let oldIndex = try String(contentsOf: packs.appendingPathComponent(oldID + ".json"), encoding: .utf8) + try Data(oldIndex.replacingOccurrences(of: oldID, with: newID).utf8).write(to: packs.appendingPathComponent(newID + ".json")) + let manifestURL = directory.appendingPathComponent("snapshots/\(second.snapshotId.uuidString).json") + var manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: manifestURL)) as? [String: Any]) + var node = try #require(manifest["treeRoot"] as? [String: Any]) + let treeID = try #require(node["treeHash"] as? String) + let oldTree = try String(contentsOf: directory.appendingPathComponent("trees/" + treeID), encoding: .utf8) + let newTree = Data(oldTree.replacingOccurrences(of: oldID, with: newID).utf8) + let newTreeID = SHA256.hexDigest(of: newTree) + try newTree.write(to: directory.appendingPathComponent("trees/" + newTreeID)) + node["treeHash"] = newTreeID + manifest["treeRoot"] = node + try JSONSerialization.data(withJSONObject: manifest).write(to: manifestURL) + let reopened = FileCheckpointStore(rootDirectory: root) + let dry = try await reopened.compact(workspaceId: id, retaining: .all, dryRun: true) + let result = try await reopened.compact(workspaceId: id, retaining: .all, dryRun: false) + #expect(result.report.after == dry.report.after) + #expect(try await reopened.storageStatistics(workspaceId: id) == dry.report.after) + #expect(result.report.after.blobCount == 100) + for checkpoint in [first, second] { + #expect(try await reopened.loadSnapshot(id: checkpoint.snapshotId, workspaceId: id)?.entry == snapshot(count: 100).entry) + } + } + + @Test(arguments: [false, true]) + func `interrupted capture never publishes incomplete blobs or trees`(failPublication: Bool) async throws { + let root = try TestSupport.temporaryDirectory("InterruptedPack") + defer { TestSupport.remove(root) } + let id = UUID() + let base = InMemoryFileSystem() + for i in 0..<100 { + try await base.writeFile(path: path(i), data: content(i), append: false) + } + let fs = HookableFileSystem(base: base) + let store = FileCheckpointStore(rootDirectory: root) + try await store.prepare(workspaceId: id) + let snapshots = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots") + await Self.arm(fs, reads: 80) { + if failPublication { + // Fail the final root-manifest write, after blobs/trees were staged. + try FileManager.default.removeItem(at: snapshots) + try Data("blocked".utf8).write(to: snapshots) + } else { throw TestFileSystemError.injected } + } + await #expect(throws: (any Error).self) { _ = try await store.captureRevision(from: fs, draft: draft(id)) } + if failPublication { + try FileManager.default.removeItem(at: snapshots) + try FileManager.default.createDirectory(at: snapshots, withIntermediateDirectories: true) + } + let reader = FileCheckpointStore(rootDirectory: root) + #expect(try await reader.listCheckpoints(workspaceId: id).isEmpty) + #expect(try FileManager.default.contentsOfDirectory(atPath: snapshots.path).isEmpty) + let swept = try await reader.compact(workspaceId: id, retaining: .all, dryRun: false) + #expect(swept.report.after == .empty) + #expect(try await reader.storageStatistics(workspaceId: id) == .empty) + let recovered = try await reader.captureRevision(from: base, draft: draft(id)) + #expect(try await reader.loadSnapshot(id: recovered.snapshotId, workspaceId: id)?.entry == snapshot(count: 100).entry) + } + + @Test func `name trees deduplicate relocated subtrees and reject invalid names`() async throws { + let root = try TestSupport.temporaryDirectory("RelativeTrees") + defer { TestSupport.remove(root) } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root, cacheByteLimit: 4096) + let absent = UUID() + do { + _ = try await store.resolveSnapshotFile(id: absent, workspaceId: id, path: "/missing") + Issue.record("missing workspace unexpectedly resolved a file") + } catch { + if case let WorkspaceError.revisionDataNotFound(actual) = error { + #expect(actual == absent) + } else { Issue.record("unexpected missing-workspace error: \(error)") } + } + + func tree(_ name: String) -> Snapshot { + let parent = WorkspacePath.root.appending(name) + return Snapshot(rootPath: .root, entry: .directory(.init(path: .root, permissions: .defaultDirectory, children: [ + .directory(.init(path: parent, permissions: .defaultDirectory, children: [ + .directory(.init(path: parent.appending("deep"), permissions: .defaultDirectory, children: [ + .file(.init(path: parent.appending("deep/file"), data: Data("same".utf8), permissions: .defaultFile)) + ])) + ])) + ]))) + } + let first = try await store.saveRevision(tree("before"), draft: draft(id)) + let directory = root.appendingPathComponent(id.uuidString) + let trees = directory.appendingPathComponent("trees") + let original = try artifacts(trees) + let second = try await store.saveRevision(tree("after"), draft: draft(id, parent: first.id)) + #expect(try artifacts(trees).count == original.count + 1) // only root changes on rename + for (checkpoint, name) in [(first, "before"), (second, "after")] { + let path = WorkspacePath(normalizing: "/\(name)/deep/file") + #expect(try await store.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: id, path: path, offset: 0, length: nil) == Data("same".utf8)) + } + // Independent malformed tree objects: each hash is valid, only the name is invalid. + let manifestURL = directory.appendingPathComponent("snapshots/\(second.snapshotId).json") + var manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: manifestURL)) as? [String: Any]) + var node = try #require(manifest["treeRoot"] as? [String: Any]) + for name in ["", ".", "..", "a/b", "a/\u{301}b", "a\0b"] { + let bytes = try JSONSerialization.data(withJSONObject: [["name": name, "kind": "directory"]]) + let hash = SHA256.hexDigest(of: bytes) + try bytes.write(to: trees.appendingPathComponent(hash)) + node["treeHash"] = hash + manifest["treeRoot"] = node + try JSONSerialization.data(withJSONObject: manifest).write(to: manifestURL) + await #expect(throws: WorkspaceError.self) { + _ = try await store.snapshotEntry(id: second.snapshotId, workspaceId: id, path: .root, includeChildren: true) + } + } + } + + @Test func `LRU refreshes hits and enforces decoded byte budget`() { + var cache = CostLRU(limit: 10) + cache.insert(1, for: "a", cost: 4) + cache.insert(2, for: "b", cost: 4) + #expect(cache.value(for: "a") == 1) + cache.insert(3, for: "c", cost: 4) + #expect(cache.value(for: "b") == nil) // hit promoted a over b + #expect(cache.value(for: "a") == 1) + #expect(cache.value(for: "c") == 3) + #expect(cache.cost == 8) + cache.insert(4, for: "a", cost: 7) + #expect(cache.value(for: "c") == nil) + #expect(cache.cost == 7) + cache.insert(5, for: "large", cost: 11) + #expect(cache.value(for: "large") == nil) + #expect(cache.value(for: "a") == 4) + cache.removeAll() + #expect(cache.cost == 0 && cache.count == 0) + } + + @Test func `durable orphan root is swept after metadata publication fails`() async throws { + let root = try TestSupport.temporaryDirectory("OrphanRoot") + defer { TestSupport.remove(root) } + let id = UUID() + let base = InMemoryFileSystem() + try await base.writeFile(path: "/note", data: Data("complete".utf8), append: false) + let fs = HookableFileSystem(base: base) + let store = FileCheckpointStore(rootDirectory: root) + try await store.prepare(workspaceId: id) + let directory = root.appendingPathComponent(id.uuidString) + let checkpoints = directory.appendingPathComponent("checkpoints") + await fs.onRead { _ in + try FileManager.default.removeItem(at: checkpoints) + try Data("blocked".utf8).write(to: checkpoints) + } + await #expect(throws: (any Error).self) { _ = try await store.captureRevision(from: fs, draft: draft(id)) } + #expect(try artifacts(directory.appendingPathComponent("snapshots")).count == 1) + try FileManager.default.removeItem(at: checkpoints) + try FileManager.default.createDirectory(at: checkpoints, withIntermediateDirectories: true) + #expect(try await store.listCheckpoints(workspaceId: id).isEmpty) + let dry = try await store.compact(workspaceId: id, retaining: .all, dryRun: true) + #expect(dry.report.before.snapshotCount == 1) + let swept = try await store.compact(workspaceId: id, retaining: .all, dryRun: false) + #expect(swept.report.after == .empty) + #expect(try artifacts(directory.appendingPathComponent("snapshots")).isEmpty) + } + + @Test func `failed capture after pack rollover sweeps indexed and unindexed packs`() async throws { + let root = try TestSupport.temporaryDirectory("RolloverFailure") + defer { TestSupport.remove(root) } + let id = UUID() + let base = InMemoryFileSystem() + for i in 0..<55 { + try await base.writeFile(path: path(i), data: Data(repeating: UInt8(i), count: 1024 * 1024), append: false) + } + let fs = HookableFileSystem(base: base) + let store = FileCheckpointStore(rootDirectory: root, cacheByteLimit: 8192) + await Self.arm(fs, reads: 53) { throw TestFileSystemError.injected } + await #expect(throws: (any Error).self) { _ = try await store.captureRevision(from: fs, draft: draft(id)) } + let packs = root.appendingPathComponent(id.uuidString).appendingPathComponent("packs") + let staged = try artifacts(packs) + #expect(staged.keys.contains { $0.hasSuffix(".json") }) + #expect(staged.keys.contains { $0.hasSuffix(".pack") && staged[String($0.dropLast(5)) + ".json"] == nil }) + try Data("pending".utf8).write(to: packs.appendingPathComponent(".pending-abandoned.pack")) + let dry = try await store.compact(workspaceId: id, retaining: .all, dryRun: true) + let swept = try await store.compact(workspaceId: id, retaining: .all, dryRun: false) + #expect(swept.report.after == dry.report.after) + #expect(swept.report.after == .empty) + #expect(try artifacts(packs).isEmpty) + // Reuse the same writer/catalog after sweeping; no hash may resolve to a deleted pack. + let recovered = try await store.captureRevision(from: base, draft: draft(id)) + let reopened = FileCheckpointStore(rootDirectory: root) + for i in 0..<55 { + #expect(try await reopened.readSnapshotFile(id: recovered.snapshotId, workspaceId: id, path: path(i), offset: 0, length: nil) == Data(repeating: UInt8(i), count: 1024 * 1024)) + } + } + + @Test(arguments: [0, 1, 3]) + func `compaction resumes after generation advance and partial object deletion`(successfulDeletes: Int) async throws { + let root = try TestSupport.temporaryDirectory("DeletionFailure") + defer { TestSupport.remove(root) } + let id = UUID() + let writer = FileCheckpointStore(rootDirectory: root) + let first = try await writer.saveRevision(snapshot(count: 100), draft: draft(id)) + let kept = snapshot(count: 10, start: 40) + let last = try await writer.saveRevision(kept, draft: draft(id, parent: first.id)) + let directory = root.appendingPathComponent(id.uuidString) + let token = directory.appendingPathComponent("packs.generation") + let generation = try Data(contentsOf: token) + let manager = DeletionFailureManager(successfulDeletes: successfulDeletes) + let failing = FileCheckpointStore(rootDirectory: root, fileManager: manager) + await #expect(throws: TestFileSystemError.self) { + _ = try await failing.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: false) + } + #expect(try Data(contentsOf: token) != generation) + let reader = FileCheckpointStore(rootDirectory: root) + #expect(try await reader.loadSnapshot(id: last.snapshotId, workspaceId: id)?.entry == kept.entry) + let dry = try await reader.compact(workspaceId: id, retaining: .all, dryRun: true) + let result = try await reader.compact(workspaceId: id, retaining: .all, dryRun: false) + #expect(result.report.after == dry.report.after) + #expect(try await reader.storageStatistics(workspaceId: id) == result.report.after) + let recovered = try await writer.saveRevision(snapshot(count: 100), draft: draft(id, parent: last.id)) + #expect(try await reader.loadSnapshot(id: recovered.snapshotId, workspaceId: id)?.entry == snapshot(count: 100).entry) + } + + private final class DeletionFailureManager: FileManager, @unchecked Sendable { + private let lock = NSLock() + private var remaining: Int + init(successfulDeletes: Int) { remaining = successfulDeletes; super.init() } + override func removeItem(at url: URL) throws { + let directory = url.deletingLastPathComponent().lastPathComponent + if directory == "trees" || directory == "packs" { + try lock.withLock { + if remaining == 0 { throw TestFileSystemError.injected } + remaining -= 1 + } + } + try super.removeItem(at: url) + } + } + + @Test func `canonically equivalent sibling names remain distinct across saves and captures`() async throws { + let root = try TestSupport.temporaryDirectory("ByteNames") + defer { TestSupport.remove(root) } + let id = UUID() + let nfc = WorkspacePath(normalizing: "/caf\u{E9}") + let nfd = WorkspacePath(normalizing: "/cafe\u{301}") + func snapshot(_ nfdBytes: Data) -> Snapshot { + Snapshot(rootPath: .root, entry: .directory(.init(path: .root, permissions: .defaultDirectory, children: [ + .file(.init(path: nfc, data: Data([0, 1]), permissions: .defaultFile)), + .file(.init(path: nfd, data: nfdBytes, permissions: .defaultFile)) + ]))) + } + let store = FileCheckpointStore(rootDirectory: root) + // Independent saved roots exercise persistence without the legacy snapshot + // diff's String-keyed map, which is fixed by the separate security track. + let first = try await store.saveRevision(snapshot(Data([0, 2])), draft: draft(id)) + let second = try await store.saveRevision(snapshot(Data([0, 2])), draft: draft(id)) + let trees = root.appendingPathComponent(id.uuidString).appendingPathComponent("trees") + let original = try artifacts(trees) + let unchanged = try await store.captureRevision( + from: ByteNameFileSystem(snapshot(Data([0, 2]))), draft: draft(id, parent: second.id) + ) + #expect(unchanged.summary.changedPathCount == 0) + let afterMetadataCapture = try artifacts(trees) + let again = try await store.captureRevision( + from: ByteNameFileSystem(snapshot(Data([0, 2]))), draft: draft(id, parent: unchanged.id) + ) + #expect(again.summary.changedPathCount == 0) + #expect(try artifacts(trees) == afterMetadataCapture) + let changed = try await store.captureRevision( + from: ByteNameFileSystem(snapshot(Data([0, 3]))), draft: draft(id, parent: again.id) + ) + #expect(changed.summary.touchedPaths.map { Array($0.string.utf8) } == [Array(nfd.string.utf8)]) + #expect(try artifacts(trees).count == afterMetadataCapture.count + 1) + #expect(!original.isEmpty) + // Both hot and cold path lookups must select the byte-exact sibling. + for reader in [store, FileCheckpointStore(rootDirectory: root)] { + for checkpoint in [first, second, unchanged, again, changed] { + #expect(try await reader.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: id, path: nfc, offset: 0, length: nil) == Data([0, 1])) + let expected = checkpoint.id == changed.id ? Data([0, 3]) : Data([0, 2]) + #expect(try await reader.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: id, path: nfd, offset: 0, length: nil) == expected) + } + } + } + + @Test func `parented saves preserve byte-only symlink target changes`() async throws { + let root = try TestSupport.temporaryDirectory("ByteSymlinkTargets") + defer { TestSupport.remove(root) } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + func snapshot(_ target: String) -> Snapshot { + Snapshot(rootPath: .root, entry: .directory(.init( + path: .root, permissions: .defaultDirectory, children: [ + .symlink(.init(path: "/alias", target: target, permissions: POSIXPermissions(0o777))) + ] + ))) + } + let before = try await store.saveRevision(snapshot("/caf\u{e9}"), draft: draft(id)) + let after = try await store.saveRevision(snapshot("/cafe\u{301}"), draft: draft(id, parent: before.id)) + #expect(after.summary.touchedPaths == ["/alias"]) + for reader in [store, FileCheckpointStore(rootDirectory: root)] { + let saved = try #require(await reader.loadSnapshot(id: after.snapshotId, workspaceId: id)) + #expect(saved.entry == snapshot("/cafe\u{301}").entry) + } + } + + /// Deliberately bypasses host filesystem normalization and WorkspacePath's + /// pre-security-patch Hashable conformance. Directory order is NFC then NFD. + private final class ByteNameFileSystem: FileSystem, Sendable { + let files: [Snapshot.File] + init(_ snapshot: Snapshot) { + guard case let .directory(directory) = snapshot.entry else { preconditionFailure() } + files = directory.children.compactMap { if case let .file(file) = $0 { file } else { nil } } + } + private func file(_ path: WorkspacePath) throws -> Snapshot.File { + guard let file = files.first(where: { $0.path.string.utf8.elementsEqual(path.string.utf8) }) else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) + } + return file + } + func stat(path: WorkspacePath) async throws -> FileInfo { + if path.isRoot { return FileInfo(path: path, kind: .directory, size: 0, permissions: .defaultDirectory, modificationDate: nil) } + let value = try file(path) + return FileInfo(path: path, kind: .file, size: UInt64(value.data.count), permissions: value.permissions, modificationDate: nil) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + guard path.isRoot else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) } + return files.map { DirectoryEntry(name: $0.path.name, info: FileInfo(path: $0.path, kind: .file, size: UInt64($0.data.count), permissions: $0.permissions, modificationDate: nil)) } + } + func readFile(path: WorkspacePath) async throws -> Data { try file(path).data } + func exists(path: WorkspacePath) async -> Bool { path.isRoot || (try? file(path)) != nil } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { throw WorkspaceError.unsupported("test filesystem glob") } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { throw WorkspaceError.readOnly } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func remove(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { throw WorkspaceError.readOnly } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + } + + private static func arm(_ fs: HookableFileSystem, reads: Int, action: @escaping @Sendable () throws -> Void) async { + await fs.onRead { _ in + if reads == 1 { try action() } + else { await arm(fs, reads: reads - 1, action: action) } + } + } + + private func artifacts(_ directory: URL) throws -> [String: Data] { + try Dictionary(uniqueKeysWithValues: FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil).map { ($0.lastPathComponent, try Data(contentsOf: $0)) }) + } + private func content(_ i: Int) -> Data { + // Exercise direct large-blob writes as well as buffered tiny records. + if i == 99 { return Data(repeating: 0, count: 1024 * 1024 + 17) } + return Data(String(format: "content-%03d\n", i).utf8) + } + private func path(_ i: Int) -> WorkspacePath { WorkspacePath(normalizing: String(format: "/file%03d", i)) } + private func snapshot(count: Int, start: Int = 0) -> Snapshot { + Snapshot(rootPath: .root, entry: .directory(.init(path: .root, permissions: .defaultDirectory, + children: (start..<(start + count)).map { .file(.init(path: path($0), data: content($0), permissions: .defaultFile)) }))) + } + private func draft(_ id: UUID, parent: UUID? = nil) -> CheckpointDraft { + CheckpointDraft(workspaceID: id, label: nil, preferredParentID: parent, origin: .manual) + } + + // Verbatim cad29c4 manifest types; only the outer type name differs. + private struct V1SnapshotManifest: Codable, Sendable { + struct Node: Codable, Sendable { + enum Kind: String, Codable { + case file, directory, symlink, missing + } + + var kind: Kind + var path: WorkspacePath + var permissions: POSIXPermissions? + var contentHash: String? + var contentSize: UInt64? = nil + var modificationDate: Date? = nil + var symlinkTarget: String? + var children: [Node]? + } + + var version: Int + var id: UUID + var rootPath: WorkspacePath + var root: Node + // Older manifests and materialized snapshots have no trustworthy capture timestamp. + var captureStartedAt: Date? = nil + } + +} diff --git a/Tests/WorkspaceTests/Fixtures/README.md b/Tests/WorkspaceTests/Fixtures/README.md new file mode 100644 index 0000000..0f55119 --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/README.md @@ -0,0 +1,7 @@ +`v1/` was generated using the original `FileCheckpointStore` from commit +`cad29c479e28ce905d4e10cf3675c59838c77928`, before changing the storage format. +It contains two checkpoints of `/note`, `/nested/deep/file`, and a relative symlink +`/link -> nested/deep/file`, an append-only mutation log, and the +`mutations.sequence.json` high-water-mark/retraction-epoch sidecar. The first +checkpoint is labeled `legacy`; the second is unlabeled. The bytes are copied +verbatim so compatibility tests exercise persisted data from the old writer. diff --git a/Tests/WorkspaceTests/Fixtures/v1/.10000000-0000-0000-0000-000000000001.format.lock b/Tests/WorkspaceTests/Fixtures/v1/.10000000-0000-0000-0000-000000000001.format.lock new file mode 100644 index 0000000..e69de29 diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 new file mode 100644 index 0000000..0ad6ad6 --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 @@ -0,0 +1 @@ +nested legacy diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 new file mode 100644 index 0000000..a8237ec --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 @@ -0,0 +1 @@ +legacy checkpoint diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 new file mode 100644 index 0000000..b41fb5d --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 @@ -0,0 +1 @@ +legacy updated diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json new file mode 100644 index 0000000..fbbc724 --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json @@ -0,0 +1,33 @@ +{ + "createdAt" : 814174270.20685, + "firstMutationSequence" : 1, + "id" : "669D476B-6EFC-46E0-A1FB-8155C3DA8121", + "label" : "legacy", + "lastMutationSequence" : 1, + "mutationCursor" : 1, + "origin" : { + "manual" : { + + } + }, + "snapshotId" : "5D7DC62F-9749-41E4-8BCE-90306A25C30C", + "summary" : { + "changedPathCount" : 6, + "hasTextChanges" : true, + "statistics" : { + "additions" : 2, + "changedFileCount" : 2, + "changedPathCount" : 6, + "deletions" : 0 + }, + "touchedPaths" : [ + "\/", + "\/link", + "\/nested", + "\/nested\/deep", + "\/nested\/deep\/file", + "\/note" + ] + }, + "workspaceID" : "10000000-0000-0000-0000-000000000001" +} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json new file mode 100644 index 0000000..1c645cd --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json @@ -0,0 +1,28 @@ +{ + "createdAt" : 814174270.213568, + "firstMutationSequence" : 2, + "id" : "8BB89064-AA2F-4516-BCE8-5D6D2FE4E092", + "lastMutationSequence" : 2, + "mutationCursor" : 2, + "origin" : { + "manual" : { + + } + }, + "parentID" : "669D476B-6EFC-46E0-A1FB-8155C3DA8121", + "snapshotId" : "EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76", + "summary" : { + "changedPathCount" : 1, + "hasTextChanges" : true, + "statistics" : { + "additions" : 1, + "changedFileCount" : 1, + "changedPathCount" : 1, + "deletions" : 1 + }, + "touchedPaths" : [ + "\/note" + ] + }, + "workspaceID" : "10000000-0000-0000-0000-000000000001" +} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json new file mode 100644 index 0000000..2da5380 --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json @@ -0,0 +1,3 @@ +{ + "version" : 1 +} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/lifecycle.lock b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/lifecycle.lock new file mode 100644 index 0000000..e69de29 diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl new file mode 100644 index 0000000..2855807 --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl @@ -0,0 +1,2 @@ +{"operation":"edit","changes":{"changes":[],"textDiffOmissions":[]},"workspaceID":"10000000-0000-0000-0000-000000000001","timestamp":814174270.19686,"sequence":1} +{"workspaceID":"10000000-0000-0000-0000-000000000001","changes":{"changes":[],"textDiffOmissions":[]},"timestamp":814174270.210703,"operation":"edit","sequence":2} diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.lock b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.lock new file mode 100644 index 0000000..e69de29 diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json new file mode 100644 index 0000000..e29520b --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json @@ -0,0 +1,4 @@ +{ + "highWaterMark" : 2, + "retractionEpoch" : 0 +} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json new file mode 100644 index 0000000..3532972 --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json @@ -0,0 +1,65 @@ +{ + "captureStartedAt" : 814174270.205489, + "id" : "5D7DC62F-9749-41E4-8BCE-90306A25C30C", + "root" : { + "children" : [ + { + "kind" : "symlink", + "modificationDate" : 814174270.196814, + "path" : "\/link", + "permissions" : { + "rawValue" : 511 + }, + "symlinkTarget" : "nested\/deep\/file" + }, + { + "children" : [ + { + "children" : [ + { + "contentHash" : "3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47", + "contentSize" : 14, + "kind" : "file", + "modificationDate" : 814174270.196806, + "path" : "\/nested\/deep\/file", + "permissions" : { + "rawValue" : 420 + } + } + ], + "kind" : "directory", + "modificationDate" : 814174270.196807, + "path" : "\/nested\/deep", + "permissions" : { + "rawValue" : 493 + } + } + ], + "kind" : "directory", + "modificationDate" : 814174270.196769, + "path" : "\/nested", + "permissions" : { + "rawValue" : 493 + } + }, + { + "contentHash" : "47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9", + "contentSize" : 18, + "kind" : "file", + "modificationDate" : 814174270.196821, + "path" : "\/note", + "permissions" : { + "rawValue" : 420 + } + } + ], + "kind" : "directory", + "modificationDate" : 814174270.196822, + "path" : "\/", + "permissions" : { + "rawValue" : 493 + } + }, + "rootPath" : "\/", + "version" : 2 +} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json new file mode 100644 index 0000000..48515db --- /dev/null +++ b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json @@ -0,0 +1,65 @@ +{ + "captureStartedAt" : 814174270.212788, + "id" : "EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76", + "root" : { + "children" : [ + { + "kind" : "symlink", + "modificationDate" : 814174270.196814, + "path" : "\/link", + "permissions" : { + "rawValue" : 511 + }, + "symlinkTarget" : "nested\/deep\/file" + }, + { + "children" : [ + { + "children" : [ + { + "contentHash" : "3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47", + "contentSize" : 14, + "kind" : "file", + "modificationDate" : 814174270.196806, + "path" : "\/nested\/deep\/file", + "permissions" : { + "rawValue" : 420 + } + } + ], + "kind" : "directory", + "modificationDate" : 814174270.196807, + "path" : "\/nested\/deep", + "permissions" : { + "rawValue" : 493 + } + } + ], + "kind" : "directory", + "modificationDate" : 814174270.196769, + "path" : "\/nested", + "permissions" : { + "rawValue" : 493 + } + }, + { + "contentHash" : "646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2", + "contentSize" : 15, + "kind" : "file", + "modificationDate" : 814174270.2107, + "path" : "\/note", + "permissions" : { + "rawValue" : 420 + } + } + ], + "kind" : "directory", + "modificationDate" : 814174270.210701, + "path" : "\/", + "permissions" : { + "rawValue" : 493 + } + }, + "rootPath" : "\/", + "version" : 2 +} \ No newline at end of file diff --git a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift index 4870a96..8beae45 100644 --- a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift +++ b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift @@ -200,9 +200,15 @@ struct PathIdentityRegressionTests { } } - @Test - func symlinkTargetChangesAreByteExact() async throws { - let workspace = Workspace() + @Test(arguments: [false, true]) + func symlinkTargetChangesAreByteExact(diskStore: Bool) async throws { + let root = try TestSupport.temporaryDirectory("IdentitySymlink") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: diskStore ? .directory(root) : .memory) + try await workspace.createDirectory(nfc) + try await workspace.createDirectory(nfd) + try await workspace.writeData(Data([1]), to: nfc.appending("\u{301}file")) + try await workspace.writeData(Data([2]), to: nfd.appending("\u{301}file")) try await workspace.createSymbolicLink("/alias", target: nfc.string) let before = try await workspace.createCheckpoint() try await workspace.remove("/alias") @@ -210,6 +216,9 @@ struct PathIdentityRegressionTests { #expect(changed.touchedPaths == ["/alias"]) let after = try await workspace.createCheckpoint() #expect(try await workspace.diff(from: .checkpoint(before.id), to: .checkpoint(after.id)).touchedPaths == ["/alias"]) + for (checkpoint, byte) in [(before, UInt8(1)), (after, UInt8(2))] { + #expect(try await workspace.readData(from: "/alias/\u{301}file", at: .checkpoint(checkpoint.id)) == Data([byte])) + } } #if canImport(Darwin) From 75055e91429dd1aa3875baa97c6e171683ffcfb2 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 00:59:31 -0700 Subject: [PATCH 15/34] Walk trees in one pass and make search byte-oriented LocalFileSystem gains a package-internal bulk traversal: one walk per call holds directory descriptors for its duration (capped at 32 levels, then an identity-checked per-path fallback), enumerates entries with lstat semantics relative to the parent descriptor, and reads files relative to it. Snapshot capture, search, glob and LimitedFileSystem accounting use it; wrappers forward it or fall back, and AuthorizedFileSystem keeps per-entry authorization. readdir names are the stored spelling, so walks take them without a spelling recheck, and children are ordered by UTF-8 bytes to match WorkspacePath ordering. The resolve fast path opens ancestors directly through spelling-verified directory descriptors. Search compiles its include/exclude selection into the single content walk, detects binary content with memchr, matches ASCII literals on bytes with an ASCII case-fold path, runs certified line-local regexes in one pass, and builds line strings only for matches and context; results are identical to the previous algorithm. Large reads allocate from the observed size and grow geometrically. On the medium benchmark fixture, search drops from 2.9 s to 0.25 s and preview from 670 ms to 270 ms. --- .../CountingFileSystem.swift | 27 +- README.md | 15 +- .../Workspace/FS/FileSystemTraversal.swift | 94 +++++ Sources/Workspace/FS/LimitedFileSystem.swift | 32 +- Sources/Workspace/FS/LocalFileSystem.swift | 249 +++++++++--- Sources/Workspace/FS/SandboxFileSystem.swift | 7 +- .../FS/SecurityScopedFileSystem.swift | 7 +- Sources/Workspace/Snapshot.swift | 21 + .../Workspace/Support/FileSelectionPlan.swift | 63 +++ Sources/Workspace/Support/SearchText.swift | 182 +++++++++ Sources/Workspace/Support/WorkspacePath.swift | 13 +- Sources/Workspace/Workspace+Edits.swift | 13 +- Sources/Workspace/WorkspaceSearch.swift | 65 ++-- .../FileSystemTraversalTests.swift | 358 ++++++++++++++++++ .../PathIdentityRegressionTests.swift | 27 +- .../SearchEquivalenceTests.swift | 237 ++++++++++++ 16 files changed, 1290 insertions(+), 120 deletions(-) create mode 100644 Sources/Workspace/FS/FileSystemTraversal.swift create mode 100644 Sources/Workspace/Support/FileSelectionPlan.swift create mode 100644 Sources/Workspace/Support/SearchText.swift create mode 100644 Tests/WorkspaceTests/FileSystemTraversalTests.swift create mode 100644 Tests/WorkspaceTests/SearchEquivalenceTests.swift diff --git a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift index c8fdf6c..f82ba60 100644 --- a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift +++ b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift @@ -13,7 +13,7 @@ struct IOCounts: Codable, Equatable, Sendable { } /// A lock only protects counter updates; no lock is held across an await. -class CountingFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +class CountingFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { let base: any FileSystem private let lock = NSLock() private var counts = IOCounts() @@ -27,6 +27,31 @@ class CountingFileSystem: FileSystem, PotentialPathResolvingFileSystem, @uncheck if file { counts.filesRead += 1 } } } + nonisolated package var supportsScopedWalk: Bool { base.supportsScopedWalk } + + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + call("walk") + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories) { info, read, readLink in + self.call("walkEntry") + if info.path != path { self.lock.withLock { self.counts.directoryEntries += 1 } } + // The callbacks remain scoped to this awaited visit; the temporary + // conversion permits forwarding Swift's nested nonescaping parameters. + try await withoutActuallyEscaping(read) { scopedRead in + try await withoutActuallyEscaping(readLink) { scopedLink in + try await visit(info, { length in + self.call(length == nil ? "readFile" : "readRange") + let data = try await scopedRead(length) + self.read(data) + return data + }, { + self.call("readSymlink") + return try await scopedLink() + }) + } + } + } + } func stat(path: WorkspacePath) async throws -> FileInfo { call("stat") return try await base.stat(path: path) diff --git a/README.md b/README.md index f14c831..608173a 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ to an equivalent spelling is rejected too; move through a distinct temporary name if a spelling change is needed. Case-only renames remain possible on case-sensitive filesystems. Removing a missing/aliased spelling remains a no-op. Symlink targets must use the stored spelling, and `resolveRealPath` -returns that spelling. Directory listings (and future bulk traversal) use names +returns that spelling. Directory listings and bulk traversal use names emitted by `readdir`, so they already carry the stored identity. On Darwin, volume policy is cached by device. APFS and HFS+ use @@ -410,6 +410,15 @@ let workspace = Workspace(fileSystem: authorized) Authorization supports one-shot, duration-limited, and session approvals plus bounded audit records. Prefix rules use path-component boundaries and require both paths to match for copy and move. +Local scans record symlinks without descending into them and retain per-entry authorization +checks when using an authorized filesystem. Scans are live reads, not atomic snapshots: +concurrent changes can affect the result or cause an error. Snapshot capture fails when a +listed child disappears before it can be inspected or read. Scoped no-follow opens reject +substituted symlinks even when they point inside the root. Deep scans bound open directory +handles by falling back to ordinary per-path operations; no handles are cached between calls. +As with other local operations, host processes can relocate open directories or mutate +hard-linked data. + ## LLM Tool Adapters `WorkspaceToolAdapter` exposes dependency-free function definitions for read, glob, search, apply, @@ -528,3 +537,7 @@ files; transient/deleted files, repeated overwrites and OS write amplification a invisible. Compare identical presets for regressions: timing ratios and exact backend count deltas are printed. Watcher errors produce a nonzero exit while retaining successful results; run outside a restrictive macOS sandbox to measure FSEvents. + +Scoped walks report `walk` and `walkEntry` counters; their content callbacks still count as +`readFile`, `readRange`, or `readSymlink`. `directoryEntries` includes visited descendants. +These are logical visits, not POSIX syscall counts; backend-internal glob walks remain opaque. diff --git a/Sources/Workspace/FS/FileSystemTraversal.swift b/Sources/Workspace/FS/FileSystemTraversal.swift new file mode 100644 index 0000000..33e9893 --- /dev/null +++ b/Sources/Workspace/FS/FileSystemTraversal.swift @@ -0,0 +1,94 @@ +import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// Visits are serial and postorder; regular files are in path order. Links are visited, +/// never descended into. A missing root produces no visits. Read callbacks are +/// nonescaping and valid only during their visit. This is a live read-only walk, +/// not an atomic snapshot: errors from children disappearing during enumeration or +/// reads propagate, except directory errors explicitly allowed by the caller's skip +/// option. Local scoped no-follow opens reject substitutions with symlinks, even +/// when the replacement link points inside the root. Very deep local subtrees fall +/// back to ordinary per-path operations with a root identity check at the handoff. +/// +/// This optional package capability keeps custom backends source-compatible and lets +/// authorization decorators retain per-entry checks through the per-path fallback. +package protocol TraversingFileSystem: FileSystem { + /// Forwarders return false when their base needs the per-path fallback. + var supportsScopedWalk: Bool { get } + nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws +} + +package typealias FileSystemWalkVisitor = @isolated(any) ( + _ info: FileInfo, + _ read: @Sendable (_ length: Int?) async throws -> Data, + _ readLink: @Sendable () async throws -> String +) async throws -> Void + +extension TraversingFileSystem { + package var supportsScopedWalk: Bool { true } +} + +extension FileSystem { + package var supportsScopedWalk: Bool { + (self as? any TraversingFileSystem)?.supportsScopedWalk ?? false + } + nonisolated(nonsending) package func walk(at path: WorkspacePath = .root, skippingUnreadableDirectories: Bool = false, + visit: FileSystemWalkVisitor) async throws { + if let traversal = self as? any TraversingFileSystem { + try await traversal.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } else { + try await walkUsingPaths(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + } + + // In particular, never unwrap AuthorizedFileSystem here: each stat, listing and + // requested content read must pass through its normal per-entry checks. + nonisolated(nonsending) package func walkUsingPaths(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + requireExisting: Bool = false, visit: FileSystemWalkVisitor) async throws { + let info: FileInfo + if requireExisting { info = try await stat(path: path) } + else { + guard let existing = try await Snapshot.infoIfPresent(at: path, on: self) else { return } + info = existing + } + if info.kind == .directory { + let children: [DirectoryEntry] + do { children = try await listDirectory(path: path) } + catch { + guard skippingUnreadableDirectories, isSkippableWalkError(error, at: path) else { throw error } + try await visit(info, { _ in throw WorkspaceError.unsupported("cannot read a directory") }, + { throw WorkspaceError.unsupported("not a symlink") }) + return + } + for (child, _) in children.map({ ($0, walkSortKey($0)) }).sorted(by: { + $0.1.utf8.lexicographicallyPrecedes($1.1.utf8) + }) { + try await walkUsingPaths(at: path.appending(child.name), + skippingUnreadableDirectories: skippingUnreadableDirectories, requireExisting: true, visit: visit) + } + } + try await visit(info, { length in + if let length { return try await self.readFile(path: path, offset: 0, length: length) } + return try await self.readFile(path: path) + }, { try await self.readSymlink(path: path) }) + } +} + +func walkSortKey(_ entry: DirectoryEntry) -> String { + walkSortKey(name: entry.name, isDirectory: entry.info.kind == .directory) +} + +func walkSortKey(name: String, isDirectory: Bool) -> String { + name + (isDirectory ? "/" : "") +} + +func isSkippableWalkError(_ error: any Error, at path: WorkspacePath) -> Bool { + let error = error as NSError + return !path.isRoot && error.domain == NSPOSIXErrorDomain + && [Int(EACCES), Int(ENOENT), Int(ENOTDIR)].contains(error.code) +} diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index b465de5..8ef7cde 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -46,7 +46,7 @@ public enum FileSystemLimitError: Error, Sendable, Equatable, CustomStringConver /// Custom backends must follow standard mutation semantics: replacement changes one entry; /// append updates aliases; parents may be created. A composition cycle (for example a Mounted /// base that mounts this same LimitedFileSystem) deadlocks while the gate is held. -public actor LimitedFileSystem: FileSystem, PotentialPathResolvingFileSystem { +public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem { private let base: any FileSystem private let pathResolver: FileSystemPathResolver public let limits: FileSystemLimits @@ -59,6 +59,13 @@ public actor LimitedFileSystem: FileSystem, PotentialPathResolvingFileSystem { self.limits = limits } + nonisolated package var supportsScopedWalk: Bool { base.supportsScopedWalk } + + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + public func stat(path: WorkspacePath) async throws -> FileInfo { try await base.stat(path: path) } public func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) @@ -195,15 +202,20 @@ public actor LimitedFileSystem: FileSystem, PotentialPathResolvingFileSystem { private func preflight(_ mutation: Mutation) async throws { var tree: [WorkspacePath: FileInfo] = [:] - var pending = [try await base.stat(path: .root)] - while let info = pending.popLast() { - let path = info.path - tree[path] = info - if info.kind == .directory { - for child in try await base.listDirectory(path: path) { - var metadata = child.info - metadata.path = path.appending(child.name) - pending.append(metadata) + if base.supportsScopedWalk { + try await base.walk { info, _, _ in tree[info.path] = info } + } else { + // Preserve metadata-only accounting for custom backends: listings already + // supply child metadata and need not permit separate child stat calls. + var pending = [try await base.stat(path: .root)] + while let info = pending.popLast() { + tree[info.path] = info + if info.kind == .directory { + for child in try await base.listDirectory(path: info.path) { + var metadata = child.info + metadata.path = info.path.appending(child.name) + pending.append(metadata) + } } } } diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index f2c1514..5f8117e 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -18,7 +18,7 @@ private typealias LocalStat = Glibc.stat /// This is a safety layer, not a sandbox: a host process can relocate an already-open directory /// outside the root, change hard-linked data, or mutate entries during recursive operations. /// Watch subscriptions still use the configured URL and are not descriptor-pinned. -public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem, TraversingFileSystem, @unchecked Sendable { /// Returns zero on success or a captured POSIX errno on failure. Injection is per-instance /// so unsupported filesystem behavior can be tested without mounts or global syscall hooks. typealias ExclusiveRename = @Sendable (Int32, String, Int32, String) -> Int32 @@ -29,6 +29,15 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem private let stateLock = NSLock() private var configuration: ConfigurationSnapshot? + #if DEBUG + private var spellingChecks = 0 + var spellingCheckCount: Int { + stateLock.lock() + defer { stateLock.unlock() } + return spellingChecks + } + #endif + private final class Descriptor: @unchecked Sendable { let value: Int32 let spellingPolicy: LocalPathSpelling.Policy @@ -60,10 +69,14 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem #endif } - private struct Entry { + private struct Entry: Sendable { let parent: Descriptor let name: String - let path: WorkspacePath + private var resolvedPath: WorkspacePath? + private var pathComponents: [String] = [] + var path: WorkspacePath { + resolvedPath ?? WorkspacePath(unchecked: "/" + pathComponents.joined(separator: "/")) + } var isVerified = false var directoryInfo: LocalStat? @@ -73,7 +86,23 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem try LocalFileSystem.validateEntryName(name) self.parent = parent self.name = name - self.path = path + self.resolvedPath = path + } + + // Ancestor opens usually never consume the path. Retain components and + // construct it only for a returned target or diagnostic. + init(parent: Descriptor, name: String, components: [String]) throws { + try LocalFileSystem.validateEntryName(name) + self.parent = parent + self.name = name + self.pathComponents = components + } + + func atPath(_ path: WorkspacePath) -> Entry { + var entry = self + entry.resolvedPath = path + entry.pathComponents = [] + return entry } // Names emitted by readdir already have stored spelling. This initializer @@ -91,7 +120,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem init(directory: Descriptor, path: WorkspacePath) { self.parent = directory self.name = "." - self.path = path + self.resolvedPath = path self.isVerified = true self.directoryInfo = directory.directoryInfo } @@ -190,6 +219,11 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } private func checkSpelling(_ entry: Entry, creating: Bool = false) throws { + #if DEBUG + stateLock.lock() + spellingChecks += 1 + stateLock.unlock() + #endif try spelling.check(parent: entry.parent.value, name: entry.name, policy: entry.parent.spellingPolicy, creating: creating) } @@ -336,7 +370,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } let final = pending.isEmpty var entry = try Entry(parent: current, name: name, - path: WorkspacePath(unchecked: "/" + (names + [name]).joined(separator: "/"))) + components: names + [name]) if missingAncestor { // Only potential-path resolution consumes unverified entries, and // only their path strings. I/O helpers require a verified Entry. @@ -352,6 +386,28 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } return entry } + // A no-follow directory open checks the type and pins an ancestor + // without a preceding lstat. APFS defers its spelling check to the + // same parent boundary as the metadata route below. + if !final { + let fd = openat(entry.parent.value, entry.name, Self.searchDirectoryFlags | O_NOFOLLOW | O_CLOEXEC) + if fd >= 0 { + current = try directoryDescriptor(fd) + if batchAPFS && current.spellingPolicy != .darwin { + throw ResolutionRetry.componentChecks + } + names.append(entry.name) + continue + } + let code = errno + guard code == ELOOP || code == ENOTDIR || code == ENOENT else { + if batchAPFS { + try verifyParent(current, names: names, configuration: config, batchAPFS: true) + _ = try verifiedEntry(entry, creating: creating) + } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(code)) + } + } var info = LocalStat() let status = fstatat(entry.parent.value, entry.name, &info, AT_SYMLINK_NOFOLLOW) let statusError = errno @@ -396,10 +452,10 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem if info.st_mode & S_IFMT == S_IFDIR { entry.directoryInfo = info } if final { return entry } if batchAPFS && !entry.isVerified { - // This is the only open of an unverified Entry. It opens a directory + // Like the fast open above, this may open an unverified directory // with O_NOFOLLOW, retains its mandatory policy, and cannot expose // data or mutate it until verifyParent succeeds at the boundary. - let fd = openat(current.value, name, Self.searchDirectoryFlags | O_NOFOLLOW | O_CLOEXEC) + let fd = openat(entry.parent.value, entry.name, Self.searchDirectoryFlags | O_NOFOLLOW | O_CLOEXEC) if fd < 0 { let error = errno try verifyParent(current, names: names, configuration: config, batchAPFS: true) @@ -498,6 +554,120 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem linkCount: type == S_IFDIR ? nil : UInt64(info.st_nlink)) } + /// Holds only the active directory ancestry, for this read-only call. Descents and + /// file opens use O_NOFOLLOW; metadata and link targets use the same parent fd. + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + let entry: Entry + let info: LocalStat + do { + let resolved = try resolve(path, followFinal: false) + entry = resolved.atPath(path) + info = try metadata(entry) + } catch { + if Snapshot.isMissingEntry(error) { return } + throw error + } + try await walkEntry(entry, info: info, depth: 0, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + + nonisolated(nonsending) private func walkEntry(_ entry: Entry, info: LocalStat, depth: Int, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + try Task.checkCancellation() + // Bound this call's retained ancestry, leaving room for unrelated operations at + // the usual 256-descriptor soft limit. The fallback resolves each operation anew. + if depth == 32 { + let current = try self.metadata(resolve(entry.path, followFinal: false)) + guard current.st_dev == info.st_dev, current.st_ino == info.st_ino else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(ESTALE)) + } + try await walkUsingPaths(at: entry.path, skippingUnreadableDirectories: skippingUnreadableDirectories, + requireExisting: true, visit: visit) + return + } + let metadata = fileInfo(info, path: entry.path) + if metadata.kind == .directory { + let directory: Descriptor + let children: [(entry: Entry, info: LocalStat, key: String)] + do { + directory = try openEntry(entry, flags: O_RDONLY | O_DIRECTORY) + children = try names(in: directory, consumingOffset: true).map { name in + let child = try Entry(parent: directory, storedName: name, path: entry.path.appending(name)) + let info = try self.metadata(child) + return (child, info, walkSortKey(name: name, isDirectory: info.st_mode & S_IFMT == S_IFDIR)) + }.sorted { $0.key.utf8.lexicographicallyPrecedes($1.key.utf8) } + } catch { + guard skippingUnreadableDirectories, isSkippableWalkError(error, at: entry.path) else { throw error } + try await visit(metadata, { _ in throw WorkspaceError.unsupported("cannot read a directory") }, + { throw WorkspaceError.unsupported("not a symlink") }) + return + } + for (child, info, _) in children { + try await walkEntry(child, info: info, depth: depth + 1, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + } + try await visit(metadata, { length in try self.readEntry(entry, length: length) }, + { try self.linkTarget(entry) }) + } + + private func openRegularFile(_ entry: Entry) throws -> (Descriptor, LocalStat) { + let fd = try openEntry(entry, flags: O_RDONLY | O_NONBLOCK) + var info = LocalStat() + try check(fstat(fd.value, &info)) + guard info.st_mode & S_IFMT == S_IFREG else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(info.st_mode & S_IFMT == S_IFDIR ? EISDIR : EINVAL)) + } + return (fd, info) + } + + private func readEntry(_ entry: Entry, offset: UInt64 = 0, length: Int?) throws -> Data { + if let length, length < 0 { throw WorkspaceError.unsupported("read length must not be negative") } + let (fd, info) = try openRegularFile(entry) + let size = UInt64(max(0, info.st_size)) + if offset > 0 { + guard offset < size else { return Data() } + if lseek(fd.value, off_t(offset), SEEK_SET) < 0 { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + } + let limit = length ?? Int.max + // Allocate for the observed size, bounded by the range and a 256 MiB initial + // allocation ceiling. Read directly into the result, then probe EOF and + // accommodate concurrent growth (or files larger than the initial allocation). + var data = Data(count: min(limit, 256 * 1_024 * 1_024, Int(clamping: size - min(offset, size)))) + var filled = 0 + while filled < limit { + if filled == data.count { + var extra = [UInt8](repeating: 0, count: min(4_096, limit - filled)) + let count = extra.withUnsafeMutableBytes { read(fd.value, $0.baseAddress!, $0.count) } + if count < 0 { + if errno == EINTR { continue } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + if count == 0 { break } + // Probe EOF before reserving more space, so unchanged small files + // do not allocate a large spare buffer. Once growth is observed, + // double the result (at least 1 MiB) and read directly into it. + let capacity = filled + min(limit - filled, max(1_024 * 1_024, filled)) + data.append(extra, count: count) + filled += count + data.count = capacity + } else { + let count = data.withUnsafeMutableBytes { + read(fd.value, $0.baseAddress!.advanced(by: filled), min($0.count - filled, Int(Int32.max))) + } + if count < 0 { + if errno == EINTR { continue } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + if count == 0 { break } + filled += count + } + } + data.count = filled + return data + } + /// See ``FileSystem/stat(path:)``. /// /// Uses lstat semantics: inspect the link itself, including dangling or external links. @@ -505,9 +675,11 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem try fileInfo(metadata(resolve(path, followFinal: false)), path: path) } - private func names(in directory: Descriptor) throws -> [String] { - // openat(".") gives an independent directory offset; dup would share it. - let fd = openat(directory.value, ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC) + private func names(in directory: Descriptor, consumingOffset: Bool = false) throws -> [String] { + // Walks own a fresh descriptor and enumerate it only once. Other callers need + // an independent offset, especially when their descriptor is the pinned root. + let fd = consumingOffset ? try duplicateDescriptor(directory.value) + : openat(directory.value, ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC) guard fd >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } @@ -538,7 +710,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem result.append(name) } } - return result.sorted { $0.utf8.lexicographicallyPrecedes($1.utf8) } + return consumingOffset ? result : result.sorted { $0.utf8.lexicographicallyPrecedes($1.utf8) } } /// See ``FileSystem/listDirectory(path:)``. @@ -553,12 +725,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem } private func readHandle(_ path: WorkspacePath) throws -> FileHandle { - let fd = try openEntry(resolve(path, followFinal: true), flags: O_RDONLY | O_NONBLOCK) - var info = LocalStat() - try check(fstat(fd.value, &info)) - guard info.st_mode & S_IFMT == S_IFREG else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(info.st_mode & S_IFMT == S_IFDIR ? EISDIR : EINVAL)) - } + let (fd, _) = try openRegularFile(resolve(path, followFinal: true)) return FileHandle(fileDescriptor: try duplicateDescriptor(fd.value), closeOnDealloc: true) } @@ -570,31 +737,12 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem /// See ``FileSystem/readFile(path:)``. public func readFile(path: WorkspacePath) async throws -> Data { - let handle = try readHandle(path) - defer { - try? handle.close() - } - return try handle.readToEnd() ?? Data() + try readEntry(resolve(path, followFinal: true), length: nil) } /// See ``FileSystem/readFile(path:offset:length:)``. public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { - if let length, length < 0 { - throw WorkspaceError.unsupported("read length must not be negative") - } - let handle = try readHandle(path) - defer { - try? handle.close() - } - let size = try handle.seekToEnd() - guard offset < size else { - return Data() - } - try handle.seek(toOffset: offset) - if let length { - return try handle.read(upToCount: length) ?? Data() - } - return try handle.readToEnd() ?? Data() + try readEntry(resolve(path, followFinal: true), offset: offset, length: length) } private actor ChunkReader { @@ -921,26 +1069,9 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem return await exists(path: pattern) ? [pattern] : [] } let regex = try NSRegularExpression(pattern: WorkspacePath.globToRegex(pattern.string)) - var paths: [WorkspacePath] = [.root] - var queue: [WorkspacePath] = [.root] - while !queue.isEmpty { - let directory = queue.removeFirst() - let entries: [DirectoryEntry] - do { - entries = try await listDirectory(path: directory) - } catch let error as NSError - where !directory.isRoot - && error.domain == NSPOSIXErrorDomain - && [Int(EACCES), Int(ENOENT), Int(ENOTDIR)].contains(error.code) - { - continue - } - for entry in entries { - paths.append(entry.info.path) - if entry.info.kind == .directory { - queue.append(entry.info.path) - } - } + var paths: [WorkspacePath] = [] + try await walk(at: .root, skippingUnreadableDirectories: true) { info, _, _ in + paths.append(info.path) } return paths.filter { regex.firstMatch(in: $0.string, range: NSRange($0.string.startIndex..., in: $0.string)) != nil diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index b22445b..7abb16a 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -1,7 +1,7 @@ import Foundation /// A disk-backed filesystem rooted in a standard app sandbox location. -public final class SandboxFileSystem: FileSystem, PotentialPathResolvingFileSystem, Sendable { +public final class SandboxFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem, Sendable { /// Supported strategies for choosing the sandbox root. public enum Root: Sendable { /// The user's documents directory. @@ -25,6 +25,11 @@ public final class SandboxFileSystem: FileSystem, PotentialPathResolvingFileSyst try backing.applyConfiguration(rootDirectory: resolvedRoot) } + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + try await backing.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { try await backing.stat(path: path) diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index faa63db..9fe13c9 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -5,7 +5,7 @@ import Foundation /// /// This adapter is intended for iOS and macOS workflows where filesystem access must be granted through /// a bookmark or document picker URL. -public final class SecurityScopedFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// The permitted mutability for the scoped filesystem. public enum AccessMode: Sendable { /// Only read operations are allowed. @@ -128,6 +128,11 @@ public final class SecurityScopedFileSystem: FileSystem, PotentialPathResolvingF } } + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + try await backing.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { try await backing.stat(path: path) diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index 4728c9d..feb2a11 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -188,12 +188,33 @@ extension Snapshot { return false } + private static func captureTree(from target: any FileSystem, at path: WorkspacePath) async throws -> Entry { + var children: [WorkspacePath: [Entry]] = [:] + var root: Entry = .missing(.init(path: path)) + try await target.walk(at: path) { info, read, readLink in + let entry: Entry + switch info.kind { + case .directory: + entry = .directory(.init(path: info.path, permissions: info.permissions, + children: (children.removeValue(forKey: info.path) ?? []).sorted { $0.path < $1.path })) + case .file: + entry = .file(.init(path: info.path, data: try await read(nil), permissions: info.permissions)) + case .symlink: + entry = .symlink(.init(path: info.path, target: try await readLink(), permissions: info.permissions)) + } + if info.path == path { root = entry } + else { children[info.path.parent, default: []].append(entry) } + } + return root + } + private static func captureEntry(from target: any FileSystem, at path: WorkspacePath) async throws -> Entry { guard let info = try await infoIfPresent(at: path, on: target) else { return .missing(.init(path: path)) } switch info.kind { case .directory: + if target.supportsScopedWalk { return try await captureTree(from: target, at: path) } let entries = try await target.listDirectory(path: path) let children = try await entries .sorted { PathName($0.name) < PathName($1.name) } diff --git a/Sources/Workspace/Support/FileSelectionPlan.swift b/Sources/Workspace/Support/FileSelectionPlan.swift new file mode 100644 index 0000000..61cb69f --- /dev/null +++ b/Sources/Workspace/Support/FileSelectionPlan.swift @@ -0,0 +1,63 @@ +import Foundation + +/// Shared include-union / exclude-subtraction semantics for edits and search. +struct FileSelectionPlan { + let root: WorkspacePath + let include: [WorkspacePath] + let exclude: [WorkspacePath] + + init(_ selection: FileSelection) { + root = selection.root + include = selection.include.map { WorkspacePath(normalizing: $0, relativeTo: selection.root) } + exclude = selection.exclude.map { WorkspacePath(normalizing: $0, relativeTo: selection.root) } + } + + func paths(on fileSystem: any FileSystem) async throws -> [WorkspacePath] { + var included = Set() + for pattern in include { + included.formUnion(try await fileSystem.glob(pattern: pattern.string, currentDirectory: root)) + } + for pattern in exclude { + included.subtract(try await fileSystem.glob(pattern: pattern.string, currentDirectory: root)) + } + return included.sorted() + } + + /// Literal includes can follow directory aliases. Keep those, and authorization + /// decorators' glob checks, on the per-path selection route. + func walkFilter(on fileSystem: any FileSystem) throws -> Filter? { + guard !include.isEmpty, fileSystem.supportsScopedWalk, + include.allSatisfy({ WorkspacePath.containsGlob($0.string) }) else { return nil } + return try Filter(include: include, exclude: exclude) + } + + struct Filter { + private enum Pattern { + case literal(WorkspacePath) + case glob(NSRegularExpression) + init(_ path: WorkspacePath) throws { + _ = try WorkspacePath(validating: path.string) + if WorkspacePath.containsGlob(path.string) { + self = .glob(try NSRegularExpression(pattern: WorkspacePath.globToRegex(path.string))) + } else { self = .literal(path) } + } + func matches(_ path: WorkspacePath, range: NSRange) -> Bool { + switch self { + case let .literal(value): return path == value + case let .glob(regex): return regex.firstMatch(in: path.string, range: range) != nil + } + } + } + private let include: [Pattern] + private let exclude: [Pattern] + init(include: [WorkspacePath], exclude: [WorkspacePath]) throws { + self.include = try include.map(Pattern.init) + self.exclude = try exclude.map(Pattern.init) + } + func contains(_ path: WorkspacePath) -> Bool { + let range = NSRange(path.string.startIndex..., in: path.string) + return include.contains { $0.matches(path, range: range) } + && !exclude.contains { $0.matches(path, range: range) } + } + } +} diff --git a/Sources/Workspace/Support/SearchText.swift b/Sources/Workspace/Support/SearchText.swift new file mode 100644 index 0000000..7af56f9 --- /dev/null +++ b/Sources/Workspace/Support/SearchText.swift @@ -0,0 +1,182 @@ +import Foundation + +/// Index bytes once; materialize only the lines that reach the output page. +struct SearchText { + let text: String + private let data: Data + private let starts: [Int] + private let ascii: Bool + var lineCount: Int { starts.count } + + init?(data: Data) { + guard !data.withUnsafeBytes({ bytes in + bytes.count > 0 && memchr(bytes.baseAddress!, 0, bytes.count) != nil + }), let text = String(data: data, encoding: .utf8) else { return nil } + // Foundation can strip the leading UTF-8 BOM during decoding. Index the + // decoded bytes so matching, line text and output byte counts agree. + let data = data.starts(with: [0xef, 0xbb, 0xbf]) && text.utf8.count == data.count - 3 + ? Data(data.dropFirst(3)) : data + self.data = data + self.text = text + var starts = [0] + var ascii = true + data.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + for index in bytes.indices { + let byte = bytes[index] + if byte == 10 { starts.append(index + 1) } + if byte >= 128 { ascii = false } + } + } + self.starts = starts + self.ascii = ascii + } + + private func end(_ index: Int) -> Int { + index + 1 < starts.count ? starts[index + 1] - 1 : data.count + } + func line(_ index: Int) -> String { + data.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + String(decoding: bytes[starts[index]..) -> UInt64 { + UInt64(end(range.upperBound - 1) - starts[range.lowerBound] - (range.count - 1)) + } + + func matchingLines(_ matcher: SearchMatcher) -> [Int] { + switch matcher.pattern { + case let .literal(value, caseSensitive): + // Foundation literal matching includes canonical Unicode equivalence. Byte + // matching is equivalent on ASCII; retain Foundation for Unicode text/patterns. + guard ascii, matcher.asciiNeedle else { + return starts.indices.filter { + line($0).range(of: value, options: caseSensitive ? [] : [.caseInsensitive]) != nil + } + } + guard !matcher.needle.contains(10) else { return [] } + return literalLines(matcher.needle, caseSensitive: caseSensitive) + case .regularExpression: + guard let regex = matcher.regex else { return [] } + let offsets: [Int] + if ascii { offsets = starts } + else { + var result = [0] + for (offset, value) in text.utf16.enumerated() where value == 10 { result.append(offset + 1) } + offsets = result + } + if matcher.lineLocalRegex { + var result: [Int] = [] + var line = 0 + var start = 0 + let length = text.utf16.count + while let hit = regex.firstMatch(in: text, range: NSRange(location: start, length: length - start)) { + while line + 1 < offsets.count, offsets[line + 1] <= hit.range.location { line += 1 } + result.append(line) + // Only one hit per line is observable. Jump over its remaining + // positions, especially for expressions that match empty strings. + guard line + 1 < offsets.count else { break } + line += 1 + start = offsets[line] + } + return result + } + // Anchors, lookarounds, flags and expressions that can consume LF need the + // original independent-line bounds. Reuse the whole String without allocating + // a String per line; opaque matching bounds preserve that contract. + let length = text.utf16.count + return offsets.indices.filter { index in + let end = index + 1 < offsets.count ? offsets[index + 1] - 1 : length + return regex.firstMatch(in: text, range: NSRange(location: offsets[index], length: end - offsets[index])) != nil + } + } + } + + private func literalLines(_ needle: [UInt8], caseSensitive: Bool) -> [Int] { + data.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + needle.withUnsafeBytes { target in + guard bytes.count >= target.count, !target.isEmpty else { return [] } + var result: [Int] = [] + var offset = 0 + var line = 0 + let last = bytes.count - target.count + while offset <= last { + if caseSensitive { + guard let found = memchr(bytes.baseAddress!.advanced(by: offset), Int32(needle[0]), last - offset + 1) else { break } + offset = bytes.baseAddress!.distance(to: found) + } else if SearchMatcher.fold(bytes[offset]) != needle[0] { + offset += 1 + continue + } + let matched: Bool + if caseSensitive { + matched = memcmp(bytes.baseAddress!.advanced(by: offset), target.baseAddress!, target.count) == 0 + } else { + matched = needle.indices.allSatisfy { SearchMatcher.fold(bytes[offset + $0]) == needle[$0] } + } + if matched { + while line + 1 < starts.count, starts[line + 1] <= offset { line += 1 } + result.append(line) + // Search reports one match per line, including overlapping occurrences. + offset = end(line) + 1 + } else { offset += 1 } + } + return result + } + } + } +} + +struct SearchMatcher { + let pattern: TextPattern + let needle: [UInt8] + let asciiNeedle: Bool + let regex: NSRegularExpression? + let lineLocalRegex: Bool + + init(_ pattern: TextPattern) throws { + self.pattern = pattern + switch pattern { + case let .literal(value, caseSensitive): + needle = caseSensitive ? Array(value.utf8) : value.utf8.map(Self.fold) + asciiNeedle = needle.allSatisfy { $0 < 128 } + regex = nil + lineLocalRegex = false + case let .regularExpression(value): + needle = [] + asciiNeedle = false + regex = try NSRegularExpression(pattern: value) + lineLocalRegex = Self.isLineLocal(value) + } + } + + static func fold(_ byte: UInt8) -> UInt8 { (65...90).contains(byte) ? byte + 32 : byte } + + /// Conservatively certify expressions whose matches cannot cross LF or inspect + /// beyond a line. Everything else uses bounded, per-line regex matching above. + private static func isLineLocal(_ pattern: String) -> Bool { + let bytes = Array(pattern.utf8) + var index = 0 + var inClass = false + while index < bytes.count { + let byte = bytes[index] + guard byte >= 32, byte < 127, byte != 94, byte != 36 else { return false } + if byte == 40, index + 1 < bytes.count, bytes[index + 1] == 63 { return false } + if byte == 91 { + // ICU POSIX classes (e.g. [[:space:]]) may include LF. + if index + 1 < bytes.count, [UInt8(58), 46, 61].contains(bytes[index + 1]) { return false } + inClass = true + } + if byte == 93 { inClass = false } + if byte == 92 { + index += 1 + guard index < bytes.count else { return false } + // Positive ASCII/Unicode word and digit classes cannot consume LF. + let escaped = bytes[index] + guard escaped == 100 || escaped == 119 || (escaped == 116 && !inClass) + || Array("\\.*+?()[]{}|-/$^".utf8).contains(escaped) else { return false } + } + index += 1 + } + return true + } +} diff --git a/Sources/Workspace/Support/WorkspacePath.swift b/Sources/Workspace/Support/WorkspacePath.swift index 5b8ffa3..3bc25ee 100644 --- a/Sources/Workspace/Support/WorkspacePath.swift +++ b/Sources/Workspace/Support/WorkspacePath.swift @@ -85,7 +85,9 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl /// The final path component, or `/` for the root path. var basename: String { - Self.basename(storage) + guard !isRoot else { return "/" } + let slash = storage.utf8.lastIndex(of: 47)! + return String(decoding: storage.utf8[storage.utf8.index(after: slash)...], as: UTF8.self) } /// The final path component, or `/` for the root. @@ -93,7 +95,8 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl /// The parent directory of the path. var dirname: WorkspacePath { - Self.dirname(storage) + guard let slash = storage.utf8.lastIndex(of: 47), slash != storage.utf8.startIndex else { return .root } + return WorkspacePath(unchecked: String(decoding: storage.utf8[.. WorkspacePath { - Self.join(self, String(component)) + let component = String(component) + if !component.isEmpty, component != ".", component != "..", !component.utf8.contains(47) { + return WorkspacePath(unchecked: (isRoot ? "/" : storage + "/") + component) + } + return Self.join(self, component) } public static func == (lhs: WorkspacePath, rhs: WorkspacePath) -> Bool { diff --git a/Sources/Workspace/Workspace+Edits.swift b/Sources/Workspace/Workspace+Edits.swift index 7cdcbc6..2570117 100644 --- a/Sources/Workspace/Workspace+Edits.swift +++ b/Sources/Workspace/Workspace+Edits.swift @@ -296,18 +296,9 @@ extension Workspace { static func selectedFiles(_ selection: FileSelection, on fileSystem: any FileSystem) async throws -> [WorkspacePath] { - var included = Set() - for pattern in selection.include { - let resolved = WorkspacePath(normalizing: pattern, relativeTo: selection.root) - included.formUnion(try await fileSystem.glob(pattern: resolved.string, currentDirectory: selection.root)) - } - var excluded = Set() - for pattern in selection.exclude { - let resolved = WorkspacePath(normalizing: pattern, relativeTo: selection.root) - excluded.formUnion(try await fileSystem.glob(pattern: resolved.string, currentDirectory: selection.root)) - } + let selected = try await FileSelectionPlan(selection).paths(on: fileSystem) var files: [WorkspacePath] = [] - for path in included.subtracting(excluded).sorted() { + for path in selected { if try await fileSystem.stat(path: path).kind == .file { files.append(path) } diff --git a/Sources/Workspace/WorkspaceSearch.swift b/Sources/Workspace/WorkspaceSearch.swift index 76088c0..882375d 100644 --- a/Sources/Workspace/WorkspaceSearch.swift +++ b/Sources/Workspace/WorkspaceSearch.swift @@ -223,9 +223,8 @@ extension Workspace { } let context = min(request.contextLines, SearchRequest.maximumContextLines) - let allFiles = try await selectedFiles(request.files, on: fileSystem) - let files = Array(allFiles.prefix(request.limits.maxFiles)) - var truncation: SearchResult.Truncation? = allFiles.count > files.count ? .fileLimit : nil + var selectedCount = 0 + var truncation: SearchResult.Truncation? var matches: [SearchResult.Match] = [] var skipped: [SearchResult.SkippedFile] = [] var searchedFileCount = 0 @@ -243,9 +242,7 @@ extension Workspace { encoder.outputFormatting = [.sortedKeys] return encoder } - let regex: NSRegularExpression? = if case let .regularExpression(pattern) = request.pattern { - try NSRegularExpression(pattern: pattern) - } else { nil } + let matcher = try SearchMatcher(request.pattern) func recordSkipped(_ path: WorkspacePath, reason: SearchResult.SkippedFile.Reason) throws { let value = SearchResult.SkippedFile(path: path, reason: reason) @@ -263,33 +260,34 @@ extension Workspace { } } - for path in files { - let info = try await fileSystem.stat(path: path) + try await visitSearchFiles(request.files, on: fileSystem) { info, read in + selectedCount += 1 + guard selectedCount <= request.limits.maxFiles else { + if truncation == nil { truncation = .fileLimit } + return + } + let path = info.path if info.size > request.limits.maxFileBytes { try recordSkipped(path, reason: .sizeLimitExceeded) - continue + return } let readLimit = Int(min(request.limits.maxFileBytes, UInt64(Int.max - 1))) + 1 - let data = try await fileSystem.readFile(path: path, offset: 0, length: readLimit) + let data = try await read(readLimit) // Recheck actual bytes in case the file grew between stat and read. if UInt64(data.count) > request.limits.maxFileBytes { try recordSkipped(path, reason: .sizeLimitExceeded) - continue + return } - guard !data.contains(0), let text = String(data: data, encoding: .utf8) else { + guard let text = SearchText(data: data) else { try recordSkipped(path, reason: .binary) - continue + return } searchedFileCount += 1 - let lines = text.utf8.split(separator: 0x0A, omittingEmptySubsequences: false) - .map { String(decoding: $0, as: UTF8.self) } - var byteOffsets: [UInt64] = [0] - for line in lines { byteOffsets.append(byteOffsets.last! + UInt64(line.utf8.count)) } - for (index, line) in lines.enumerated() where Self.matches(line, request.pattern, regex: regex) { + for index in text.matchingLines(matcher) { // Clamp each distance before arithmetic; no untrusted addition/subtraction. let lower = index - min(index, context) - let upper = index + 1 + min(lines.count - index - 1, context) - let matchBytes = byteOffsets[upper] - byteOffsets[lower] + let upper = index + 1 + min(text.lineCount - index - 1, context) + let matchBytes = text.byteCount(lower..= request.matchOffset, !pageClosed else { continue } @@ -298,10 +296,11 @@ extension Workspace { pageClosed = true continue } + let line = text.line(index) var match = SearchResult.Match( path: path, lineNumber: index + 1, line: line, - before: (lower.. Bool { - switch pattern { - case let .literal(value, caseSensitive): - return line.range(of: value, options: caseSensitive ? [] : [.caseInsensitive]) != nil - case .regularExpression: - guard let regex else { return false } - return regex.firstMatch(in: line, range: NSRange(line.startIndex.. Data) async throws -> Void + ) async throws { + let plan = FileSelectionPlan(selection) + guard let filter = try plan.walkFilter(on: fileSystem) else { + for path in try await selectedFiles(selection, on: fileSystem) { + let info = try await fileSystem.stat(path: path) + try await visit(info, { try await fileSystem.readFile(path: path, offset: 0, length: $0) }) + } + return + } + // Start at root because selection patterns may be absolute or contain '..'. + try await fileSystem.walk(skippingUnreadableDirectories: true) { info, read, _ in + if info.kind == .file, filter.contains(info.path) { try await visit(info, read) } } } } diff --git a/Tests/WorkspaceTests/FileSystemTraversalTests.swift b/Tests/WorkspaceTests/FileSystemTraversalTests.swift new file mode 100644 index 0000000..f0e7e31 --- /dev/null +++ b/Tests/WorkspaceTests/FileSystemTraversalTests.swift @@ -0,0 +1,358 @@ +import Foundation +import Testing +@testable import Workspace +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +struct FileSystemTraversalTests { + @Test + func deepCaptureBoundsDescriptors() async throws { + // Resource limits are process-wide. Re-execute only this test in a child so + // the reduced limit cannot interfere with Swift Testing's parallel suites. + let flag = "WORKSPACE_DEEP_CAPTURE_CHILD" + if ProcessInfo.processInfo.environment[flag] != "1" { + let process = Process() + process.executableURL = URL(fileURLWithPath: CommandLine.arguments[0]) + #if os(macOS) + let bundle = try #require(Bundle(for: TraversalTestBundle.self).executableURL) + process.arguments = ["--test-bundle-path", bundle.path, + "--testing-library", "swift-testing", "--filter", "deepCaptureBoundsDescriptors"] + #else + process.arguments = ["--testing-library", "swift-testing", "--filter", "deepCaptureBoundsDescriptors"] + #endif + var environment = ProcessInfo.processInfo.environment + environment[flag] = "1" + process.environment = environment + let output = Pipe() + process.standardOutput = output + process.standardError = output + try process.run() + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + let message = String(decoding: data, as: UTF8.self) + #expect(process.terminationStatus == 0, Comment(rawValue: message)) + #expect(message.contains("DeepTreeLimitChecked"), Comment(rawValue: message)) + return + } + #if canImport(Darwin) + let resource = RLIMIT_NOFILE + #else + let resource = Int32(RLIMIT_NOFILE.rawValue) + #endif + var original = rlimit() + try #require(getrlimit(resource, &original) == 0) + var reduced = original + reduced.rlim_cur = min(original.rlim_cur, 256) + try #require(setrlimit(resource, &reduced) == 0) + defer { #expect(setrlimit(resource, &original) == 0) } + let root = try TestSupport.temporaryDirectory("DeepWalk") + defer { TestSupport.remove(root) } + let path = WorkspacePath(normalizing: String(repeating: "/d", count: 300) + "/leaf") + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: path, data: Data("leaf".utf8), append: false) + let captured = try await Snapshot.capture(from: local) + let reference = try await Snapshot.capture(from: MountedFileSystem(base: local)) + #expect(captured.entry == reference.entry) + var visited = false + try await local.walk { info, read, _ in + if info.path == path { + visited = true + // Some host sandboxes do not enforce the child's soft limit. + // Independently check that most of the 256-slot budget + // remains available rather than accepting a false-positive EMFILE test. + let openCount = (Int32(0)..<1_024).filter { fcntl($0, F_GETFD) >= 0 }.count + #expect(openCount < 64) + // Leave meaningful headroom for other filesystem users while suspended + // in the deepest visit, not merely enough to finish this particular walk. + var descriptors: [Int32] = [] + defer { for fd in descriptors { close(fd) } } + for _ in 0..<100 { + let fd = open("/dev/null", O_RDONLY | O_CLOEXEC) + try #require(fd >= 0) + descriptors.append(fd) + } + #expect(try await read(nil) == Data("leaf".utf8)) + } + } + #expect(visited) + print("DeepTreeLimitChecked") + } + + @Test + func captureParityIncludesLinksModesAndSubtreeSpelling() async throws { + let container = try TestSupport.temporaryDirectory("WalkParity") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + let outside = container.appendingPathComponent("secret") + try Data("outside".utf8).write(to: outside) + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/dir/empty", recursive: true) + try await local.writeFile(path: "/dir/file", data: Data([0, 1, 255]), append: false) + try await local.setPermissions(path: "/dir/file", permissions: POSIXPermissions(0o640)) + try await local.setPermissions(path: "/dir", permissions: POSIXPermissions(0o750)) + try await local.createHardLink(path: "/hard", target: "/dir/file") + for (name, target) in [("dangling", "missing"), ("cycle", "cycle"), ("alias", "dir"), + ("escape", outside.path)] { + try FileManager.default.createSymbolicLink(atPath: root.appendingPathComponent(name).path, + withDestinationPath: target) + } + // Mounted with no mounts preserves the namespace and deliberately uses per-path capture. + let perPath = MountedFileSystem(base: local) + for path: WorkspacePath in [.root, "/dir", "/dir/file", "/dangling", "/cycle", "/escape", "/absent", "/alias/empty"] { + let bulk = try await Snapshot.capture(from: local, at: path) + let reference = try await Snapshot.capture(from: perPath, at: path) + #expect(bulk.entry == reference.entry) + #expect(bulk.entry.path == path) + } + var infos: [WorkspacePath: FileInfo] = [:] + var reads: [WorkspacePath: Data] = [:] + #if DEBUG + let spellingChecks = local.spellingCheckCount + #endif + try await local.walk { info, read, readLink in + infos[info.path] = info + if info.kind == .file { reads[info.path] = try await read(nil) } + if info.kind == .symlink { _ = try await readLink() } + } + #if DEBUG + // Readdir supplies stored names for files and directories alike. A root + // scan must not add a spelling lookup for each visited entry or read. + #expect(local.spellingCheckCount == spellingChecks) + #endif + #expect(Set(reads.keys) == ["/dir/file", "/hard"]) + #expect(reads["/hard"] == Data([0, 1, 255])) + #expect(infos["/hard"]?.identity == infos["/dir/file"]?.identity) + #expect(infos["/hard"]?.linkCount == 2) + #expect(infos["/dir/file"]?.permissions == POSIXPermissions(0o640)) + #expect(infos["/dir"]?.permissions == POSIXPermissions(0o750)) + } + + @Test + func combiningMarksCannotHidePathSeparatorsOrAuthorizationBoundaries() async throws { + let root = try TestSupport.temporaryDirectory("WalkUnicode") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let name = "\u{0301}file" + let rootPath = try WorkspacePath(name) + #expect(rootPath.string == "/" + name) + #expect(rootPath.name == name) + #expect(rootPath.parent == .root) + try #require(rootPath.components == [name]) + let path = try WorkspacePath("/private/" + name) + try await local.writeFile(path: path, data: Data("inside".utf8), append: false) + try await local.writeFile(path: rootPath, data: Data("root file".utf8), append: false) + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("alias"), + withDestinationURL: root.appendingPathComponent("private").appendingPathComponent(name)) + #expect(try await local.readFile(path: "/alias") == Data("inside".utf8)) + #expect(try await Snapshot.capture(from: local).entry == Snapshot.capture(from: MountedFileSystem(base: local)).entry) + let authorized = AuthorizedFileSystem(base: local, authorizer: RuleBasedPermissionAuthorizer( + rules: [.init(operations: [.readFile], pathPrefix: "/private", effect: .deny(message: "private"))], + defaultEffect: .allow)) + await #expect(throws: WorkspaceError.self) { _ = try await authorized.readFile(path: path) } + let canonicalRules = RuleBasedPermissionAuthorizer( + rules: [.init(operations: [.readFile], pathPrefix: "/caf\u{00e9}", effect: .deny(message: nil))], + defaultEffect: .allow) + let decision = await canonicalRules.authorize(.init(operation: .readFile, path: "/cafe\u{0301}/child")) + guard case .allow = decision else { + Issue.record("component prefixes must retain exact UTF-8 identity") + return + } + } + + @Test + func combiningMarkAncestorSwapCannotEscapeRoot() async throws { + let container = try TestSupport.temporaryDirectory("UnicodeAncestorSwap") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + let outside = container.appendingPathComponent("outside") + let local = try LocalFileSystem(root: root) + let name = "\u{0301}file" + try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true) + try Data("outside secret".utf8).write(to: outside.appendingPathComponent(name)) + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("private"), withDestinationURL: outside) + let path = try WorkspacePath("/private/" + name) + await #expect(throws: WorkspaceError.self) { _ = try await local.readFile(path: path) } + } + + @Test + func scopedReadsRespectBoundsAndReadGrowthBeyondEnumeratedSize() async throws { + let root = try TestSupport.temporaryDirectory("WalkReadBounds") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createFile(path: "/file", data: Data([1])) + let content = Data((0..<131_079).map { UInt8(truncatingIfNeeded: $0) }) + var visited = false + try await local.walk { info, read, _ in + guard info.path == "/file" else { return } + visited = true + #expect(info.size == 1) + // Grow after enumeration; a read must not treat the old size as EOF. + try content.write(to: root.appendingPathComponent("file")) + for length: Int? in [0, 1, 4096, 65_536, 200_000, nil] { + let result = try await read(length) + #expect(result == Data(content.prefix(length ?? content.count))) + } + } + #expect(visited) + for offset: UInt64 in [0, 1, 4_096, 65_535, UInt64(content.count - 1), UInt64(content.count), .max] { + for length: Int? in [0, 17, nil] { + let remainder = content.dropFirst(min(content.count, Int(clamping: offset))) + let expected = Data(remainder.prefix(length ?? remainder.count)) + #expect(try await local.readFile(path: "/file", offset: offset, length: length) == expected) + } + } + } + + @Test + func unreadableDirectoryFailsCaptureButGlobRetainsItsEntry() async throws { + guard geteuid() != 0 else { return } // root bypasses discretionary mode bits on Linux. + let root = try TestSupport.temporaryDirectory("WalkPermissions") + let blocked = root.appendingPathComponent("blocked") + defer { _ = chmod(blocked.path, 0o755); TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/blocked/child", data: Data(), append: false) + #expect(chmod(blocked.path, 0) == 0) + for fs: any FileSystem in [local, MountedFileSystem(base: local)] { + do { + _ = try await Snapshot.capture(from: fs) + Issue.record("capture must fail when a directory cannot be enumerated") + } catch { + let error = error as NSError + #expect(error.domain == NSPOSIXErrorDomain && error.code == Int(EACCES)) + } + } + #expect(try await local.glob(pattern: "**/*", currentDirectory: .root) == [.root, "/blocked"]) + } + + @Test + func walkPinsAncestorAndNextOperationResolvesAgain() async throws { + let container = try TestSupport.temporaryDirectory("WalkSwap") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + let outside = container.appendingPathComponent("outside") + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/dir/a", data: Data("inside-a".utf8), append: false) + try await local.writeFile(path: "/dir/b", data: Data("inside-b".utf8), append: false) + try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true) + try Data("secret".utf8).write(to: outside.appendingPathComponent("b")) + var readValues: [String] = [] + try await local.walk { info, read, _ in + if info.path == "/dir/a" { + try FileManager.default.moveItem(at: root.appendingPathComponent("dir"), + to: container.appendingPathComponent("relocated")) + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("dir"), + withDestinationURL: outside) + } + if info.kind == .file { readValues.append(String(decoding: try await read(nil), as: UTF8.self)) } + } + #expect(readValues == ["inside-a", "inside-b"]) + await #expect(throws: WorkspaceError.self) { + try await local.writeFile(path: "/dir/b", data: Data("bad".utf8), append: false) + } + #expect(try Data(contentsOf: outside.appendingPathComponent("b")) == Data("secret".utf8)) + #expect(try Data(contentsOf: container.appendingPathComponent("relocated/b")) == Data("inside-b".utf8)) + } + + @Test(arguments: [false, true], ["external", "internal", "missing"]) + func substitutionsBeforeOpenNeverFollowLinks(directory: Bool, replacement: String) async throws { + let container = try TestSupport.temporaryDirectory("WalkPendingSwap") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + let outside = replacement == "internal" ? root.appendingPathComponent("zz-target") + : container.appendingPathComponent("outside") + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/a", data: Data("first".utf8), append: false) + try await local.writeFile(path: directory ? "/z/file" : "/z", data: Data("inside".utf8), append: false) + try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true) + try Data("secret".utf8).write(to: outside.appendingPathComponent("file")) + var readValues: [String] = [] + do { + try await local.walk { info, read, _ in + if info.path == "/a" { + try FileManager.default.removeItem(at: root.appendingPathComponent("z")) + if replacement != "missing" { + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("z"), + withDestinationURL: directory ? outside : outside.appendingPathComponent("file")) + } + } + if info.kind == .file { readValues.append(String(decoding: try await read(nil), as: UTF8.self)) } + } + Issue.record("a no-follow open must reject the substituted entry") + } catch { + let error = error as NSError + #expect(error.domain == NSPOSIXErrorDomain) + #expect((replacement == "missing" ? [Int(ENOENT)] : [Int(ELOOP), Int(ENOTDIR)]).contains(error.code)) + } + #expect(readValues == ["first"]) + } + + @Test(arguments: [PermissionOperation.stat, .listDirectory, .readFile, .readSymlink]) + func wrappersCannotBypassPerEntryAuthorization(operation: PermissionOperation) async throws { + let root = try TestSupport.temporaryDirectory("WalkAuthorization") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/private/file", data: Data("secret".utf8), append: false) + try await local.createSymlink(path: "/private/link", target: "file") + let authorized = AuthorizedFileSystem(base: local, authorizer: PermissionAuthorizer { request in + if request.operation == operation, request.path?.string.hasPrefix("/private") == true { + return .deny(message: "per-entry denial") + } + return .allow + }) + let wrapped = LimitedFileSystem(base: authorized, limits: .init()) + do { + try await wrapped.walk { info, read, readLink in + if info.kind == .file { _ = try await read(nil) } + if info.kind == .symlink { _ = try await readLink() } + } + Issue.record("walk must enforce the denied per-entry operation") + } catch let WorkspaceError.accessDenied(actual, message) { + #expect(actual == operation.rawValue) + #expect(message == "per-entry denial") + } + } + + @Test + func emptySearchSelectionDoesNotOpenTheRootDirectory() async throws { + guard geteuid() != 0 else { return } + let root = try TestSupport.temporaryDirectory("EmptySearch") + let local = try LocalFileSystem(root: root) + defer { _ = chmod(root.path, 0o755); TestSupport.remove(root) } + #expect(chmod(root.path, 0o111) == 0) + let result = try await Workspace.search(.init(pattern: .literal("needle"), + files: .init(include: [])), on: local) + #expect(result.matches.isEmpty) + #expect(result.searchedFileCount == 0) + #expect(result.truncation == nil) + } + + @Test + func searchPreservesPathOrderingLimitsAndLiteralAliases() async throws { + let root = try TestSupport.temporaryDirectory("WalkSearch") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + for path: WorkspacePath in ["/a/file", "/a.txt", "/a-/file", "/a0", "/é", "/excluded/file"] { + try await local.writeFile(path: path, data: Data("match\nmatch".utf8), append: false) + } + try await local.createSymlink(path: "/alias", target: "a") + try await local.writeFile(path: "/0binary", data: Data([0, 1]), append: false) + try await local.writeFile(path: "/0large", data: Data(repeating: 65, count: 101), append: false) + let selections = [FileSelection(exclude: ["excluded/**"]), FileSelection(include: ["alias/file"])] + for selection in selections { + let request = SearchRequest(pattern: .literal("match", caseSensitive: true), files: selection, + limits: .init(maxFileBytes: 100, maxFiles: 3, maxMatches: 2), matchOffset: 1) + let fast = try await Workspace.search(request, on: local) + let fallback = try await Workspace.search(request, on: MountedFileSystem(base: local)) + #expect(fast == fallback) + #expect(fast.totalMatches > 0) + if selection.include != ["alias/file"] { + #expect(fast.skippedFiles.map(\.reason) == [.binary, .sizeLimitExceeded]) + } + } + } +} + +private final class TraversalTestBundle: NSObject {} diff --git a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift index 8beae45..0d631a3 100644 --- a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift +++ b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift @@ -271,10 +271,12 @@ struct PathIdentityRegressionTests { #expect(try await fs.listDirectory(path: "/dir").isEmpty) } - @Test - func middleAncestorAliasIsRejectedAcrossOpenPaths() async throws { - let root = try TestSupport.temporaryDirectory("IdentityAncestors") - defer { TestSupport.remove(root) } + @Test(arguments: [false, true]) + func middleAncestorAliasIsRejectedAcrossOpenPaths(relocateRoot: Bool) async throws { + let container = try TestSupport.temporaryDirectory("IdentityAncestors") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) guard try caseInsensitive(root) else { return } let fs = try LocalFileSystem(root: root) try await fs.createFile(path: "/one/middle/three/file", data: Data([42])) @@ -282,16 +284,33 @@ struct PathIdentityRegressionTests { let aliasFD = open(root.path + "/one/MIDDLE/three/file", O_RDONLY) #expect(aliasFD >= 0) close(aliasFD) + if relocateRoot { + // The pinned root remains usable, but its old F_GETPATH prefix no + // longer matches. This forces per-component spelling verification. + try FileManager.default.moveItem(at: root, to: container.appendingPathComponent("relocated")) + } await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: "/one/MIDDLE/three/file") } await expectPOSIX(ENOENT) { _ = try await fs.stat(path: "/one/MIDDLE/three/file") } await expectPOSIX(ENOENT) { _ = try await fs.listDirectory(path: "/one/MIDDLE/three") } await expectPOSIX(ENOENT) { _ = try await fs.resolveRealPath(path: "/one/MIDDLE/three/file") } + var aliasedVisits = 0 + try await fs.walk(at: "/one/MIDDLE/three") { _, _, _ in aliasedVisits += 1 } + #expect(aliasedVisits == 0) // Walk treats an ENOENT root as missing. await expectPOSIX(EEXIST) { try await fs.writeFile(path: "/one/MIDDLE/three/new", data: Data([9]), append: false) } try await fs.createSymlink(path: "/relative", target: "one/MIDDLE/../middle/three/file") await expectPOSIX(ENOENT) { _ = try await fs.readFile(path: "/relative") } #expect(try await fs.readFile(path: "/one/middle/three/file") == Data([42])) + var storedFiles: [WorkspacePath] = [] + try await fs.walk(at: "/one/middle/three") { info, read, _ in + if info.kind == .file { + storedFiles.append(info.path) + let data = try await read(nil) + #expect(data == Data([42])) + } + } + #expect(storedFiles == ["/one/middle/three/file"]) #expect(!(await fs.exists(path: "/one/middle/three/new"))) } diff --git a/Tests/WorkspaceTests/SearchEquivalenceTests.swift b/Tests/WorkspaceTests/SearchEquivalenceTests.swift new file mode 100644 index 0000000..b0a4b65 --- /dev/null +++ b/Tests/WorkspaceTests/SearchEquivalenceTests.swift @@ -0,0 +1,237 @@ +import Foundation +import Testing +@testable import Workspace + +struct SearchEquivalenceTests { + @Test + func unicodeFileOrderAndFileLimitMatchByteSortedFallback() async throws { + let root = try TestSupport.temporaryDirectory("SearchByteOrder") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + // UTF-8 puts the decomposed e before f; Swift String ordering puts é + // after f. Punctuation also exercises the directory name + "/" key. + let expected: [WorkspacePath] = ["/e\u{301}!", "/e\u{301}.txt", "/e\u{301}/child", "/f"] + for path in expected.reversed() { + try await local.createFile(path: path, data: Data("match".utf8)) + } + let fallback = MountedFileSystem(base: local) + for fs: any FileSystem in [local, fallback] { + var paths: [WorkspacePath] = [] + try await fs.walk { info, _, _ in + if info.kind == .file { paths.append(info.path) } + } + #expect(paths == expected) + } + for limit in 0...expected.count { + let request = SearchRequest(pattern: .literal("match"), + limits: .init(maxFileBytes: 100, maxFiles: limit, maxMatches: 10)) + let scoped = try await Workspace.search(request, on: local) + let reference = try await Workspace.search(request, on: fallback) + #expect(scoped == reference) + #expect(scoped.matches.map(\.path) == Array(expected.prefix(limit))) + #expect(scoped.truncation == (limit < expected.count ? .fileLimit : nil)) + } + } + + @Test + func optimizedSearchMatchesIndependentLineAlgorithm() async throws { + let root = try TestSupport.temporaryDirectory("SearchEquivalence") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let samples = [ + "", "\n", "start end", "TODO first\r\nsecond TODO\r\n\r\nTODO", + "aaaa\naaa\naa\na\n", "ABC abc AbC\nabc\rdef\n", + "café cafe\u{0301} CAFÉ\nStraße STRASSE\nİ i ı I\nΣ σ ς\n", + "a\u{0301}a 👩🏽‍💻 é\n終わり\n", "first\nlast", "first\nlast\n", + "one\rtwo\u{2028}three\u{2029}four\n", String(repeating: "TODO\t\"\\", count: 300), + "\u{FEFF}abc\nTODO\n", "\u{FEFF}TODO", + "\u{FEFF}", "\u{FEFF}\u{FEFF}TODO\n", "abc\u{FEFF}TODO", + ] + for (index, text) in samples.enumerated() { + try await local.writeFile(path: WorkspacePath(normalizing: "/f/\(index).txt"), data: Data(text.utf8), append: false) + } + try await local.writeFile(path: "/binary", data: Data([97, 0, 98]), append: false) + try await local.writeFile(path: "/invalid", data: Data([0xff]), append: false) + try await local.writeFile(path: "/oversized", data: Data(repeating: 65, count: 5_000), append: false) + let patterns: [TextPattern] = [ + .literal("TODO"), .literal("aa"), .literal("start"), .literal("last"), .literal("\n"), + .literal("abc", caseSensitive: false), .literal("a"), .literal("é"), .literal("café"), + .literal("CAFÉ", caseSensitive: false), .literal("strasse", caseSensitive: false), + .literal("i", caseSensitive: false), .literal("σ", caseSensitive: false), .literal("\u{FEFF}"), + .regularExpression("TODO|aa"), .regularExpression("a*"), .regularExpression("[a-z]+"), .regularExpression("[[:space:]]+"), + .regularExpression("^"), .regularExpression("$"), .regularExpression("^.*$"), + .regularExpression("(?m)^.*$"), .regularExpression("(?s)first.*last"), + .regularExpression("first\\nlast|last"), .regularExpression("[^x]+"), + .regularExpression("\\A|\\z"), .regularExpression("(?<=\\n)last|last(?!.)"), + .regularExpression("\\b\\w+\\b"), .regularExpression(".*"), .regularExpression("[\\t-z]+"), + .regularExpression("b|"), + ] + for pattern in patterns { + for context in 0...2 { + for (maxFiles, maxMatches, offset, budget) in [(30, 100, 0, nil as Int?), (8, 2, 1, nil), (30, 4, 1, 1_024), (0, 0, 0, nil)] { + let request = SearchRequest(pattern: pattern, + files: FileSelection(include: ["**/*", "f/*.txt"], exclude: ["f/3?.txt"]), contextLines: context, + limits: .init(maxFileBytes: 4_000, maxFiles: maxFiles, maxMatches: maxMatches, maxOutputBytes: budget), + matchOffset: offset) + let actual = try await Workspace.search(request, on: local) + let expected = try await Self.referenceSearch(request, on: local) + #expect(actual == expected, "pattern: \(pattern); context: \(context); limits: \(request.limits)") + } + } + } + } + + // Frozen pre-optimization algorithm from cad29c4. This intentionally uses + // independent line Strings and Foundation matching as the compatibility oracle, + // including pagination, byte budgets, skipped files and context serialization. + static func referenceSearch(_ request: SearchRequest, on fileSystem: any FileSystem) async throws -> SearchResult { + guard request.contextLines >= 0, request.limits.maxFiles >= 0, + request.limits.maxMatches >= 0, request.matchOffset >= 0 else { + throw WorkspaceError.unsupported("search limits, offset and context must not be negative") + } + if let budget = request.limits.maxOutputBytes, budget < SearchRequest.Limits.minimumOutputBytes { + throw WorkspaceError.unsupported("search output budget must be at least \(SearchRequest.Limits.minimumOutputBytes) bytes") + } + switch request.pattern { + case let .literal(value, _), let .regularExpression(value): + guard !value.isEmpty else { throw WorkspaceError.unsupported("search pattern must not be empty") } + } + + let context = min(request.contextLines, SearchRequest.maximumContextLines) + let allFiles = try await Workspace.selectedFiles(request.files, on: fileSystem) + let files = Array(allFiles.prefix(request.limits.maxFiles)) + var truncation: SearchResult.Truncation? = allFiles.count > files.count ? .fileLimit : nil + var matches: [SearchResult.Match] = [] + var skipped: [SearchResult.SkippedFile] = [] + var searchedFileCount = 0 + var totalMatches = 0 + var pageClosed = false + // Reserve more than the maximum encoded envelope (including all integer digits). + // Charge each array element's actual JSON size plus its comma. This bounds escaping, + // paths, context and skipped-file metadata, rather than just raw text bytes. + var remaining = request.limits.maxOutputBytes.map { $0 - SearchRequest.Limits.envelopeReserve } + // Skipped files cannot consume the whole page before its first matching line. + var skippedRemaining = remaining.map { $0 / 2 } + let encoder: JSONEncoder? = request.limits.maxOutputBytes.map { _ in + let encoder = JSONEncoder() + // Measurement settings must match WorkspaceToolAdapter.call's result encoder. + encoder.outputFormatting = [.sortedKeys] + return encoder + } + let regex: NSRegularExpression? = if case let .regularExpression(pattern) = request.pattern { + try NSRegularExpression(pattern: pattern) + } else { nil } + + func recordSkipped(_ path: WorkspacePath, reason: SearchResult.SkippedFile.Reason) throws { + let value = SearchResult.SkippedFile(path: path, reason: reason) + guard let encoder, let budget = remaining, let skippedBudget = skippedRemaining else { + skipped.append(value) + return + } + let cost = try encoder.encode(value).count + if cost < min(budget, skippedBudget) { + skipped.append(value) + remaining = budget - cost - 1 + skippedRemaining = skippedBudget - cost - 1 + } else { + truncation = .outputLimit + } + } + + for path in files { + let info = try await fileSystem.stat(path: path) + if info.size > request.limits.maxFileBytes { + try recordSkipped(path, reason: .sizeLimitExceeded) + continue + } + let readLimit = Int(min(request.limits.maxFileBytes, UInt64(Int.max - 1))) + 1 + let data = try await fileSystem.readFile(path: path, offset: 0, length: readLimit) + // Recheck actual bytes in case the file grew between stat and read. + if UInt64(data.count) > request.limits.maxFileBytes { + try recordSkipped(path, reason: .sizeLimitExceeded) + continue + } + guard !data.contains(0), let text = String(data: data, encoding: .utf8) else { + try recordSkipped(path, reason: .binary) + continue + } + searchedFileCount += 1 + let lines = text.utf8.split(separator: 0x0A, omittingEmptySubsequences: false) + .map { String(decoding: $0, as: UTF8.self) } + var byteOffsets: [UInt64] = [0] + for line in lines { byteOffsets.append(byteOffsets.last! + UInt64(line.utf8.count)) } + for (index, line) in lines.enumerated() where Self.matches(line, request.pattern, regex: regex) { + // Clamp each distance before arithmetic; no untrusted addition/subtraction. + let lower = index - min(index, context) + let upper = index + 1 + min(lines.count - index - 1, context) + let matchBytes = byteOffsets[upper] - byteOffsets[lower] + let ordinal = totalMatches + totalMatches += 1 + guard ordinal >= request.matchOffset, !pageClosed else { continue } + guard matches.count < request.limits.maxMatches else { + truncation = .matchLimit + pageClosed = true + continue + } + var match = SearchResult.Match( + path: path, lineNumber: index + 1, line: line, + before: (lower..= budget, matches.isEmpty { + // Guarantee progress for a single oversized match. Drop context first, + // then shorten the line at a UTF-8 boundary; file reads expose the rest. + match.truncated = true + match.before = [] + match.after = [] + cost = try encoder.encode(match).count + if cost >= budget { + match.line = "" + let overhead = try encoder.encode(match).count + if overhead < budget - 1 { + let bytes = Array(line.utf8.prefix((budget - overhead - 1) / 6)) + var end = bytes.count + while end > 0, String(bytes: bytes[.. Bool { + switch pattern { + case let .literal(value, caseSensitive): + return line.range(of: value, options: caseSensitive ? [] : [.caseInsensitive]) != nil + case .regularExpression: + guard let regex else { return false } + return regex.firstMatch(in: line, range: NSRange(line.startIndex.. Date: Tue, 20 Oct 2026 01:20:41 -0700 Subject: [PATCH 16/34] Make transactions, preview and observation scale with the change Transactions freeze an indexed base and give the draft a copy-on-write overlay. Commit finds parent-side changes from metadata against that base, applying the store's racy-mtime rule (re-reading anything modified within two seconds of the previous capture, and always under .verifyContents), and merges and rolls back only the changed paths. A cold begin keeps up to 256 MiB of the base in memory and spools the rest privately, releasing it on commit, discard or failure; later transactions re-seed from the committed checkpoint. Preview captures only the edits' affected roots and applies them to an overlay that follows the backing filesystem's semantics, so its ChangeSets match apply's. Filesystem observation keeps a metadata index built with the bulk traversal instead of a full-content snapshot, plus an LRU content cache for text diffs (ObservationPolicy). Change sources report scoped signals: an entry, a directory listing, or a recursive subtree. FSEvents and inotify map their events to the narrowest correct scope, so one external write is handled without walking the tree. Failed refreshes keep their pending changes and retry a bounded number of times, and unreadable files are reported as changes with a content-read omission. On the medium benchmark fixture, a steady transaction drops from 613 ms to 121 ms, preview from 287 ms to under 1 ms, and handling one external change from 279 ms to 58 ms. --- .../CountingFileSystem.swift | 13 +- .../WorkspaceBenchmarks/Scenarios.swift | 9 +- README.md | 67 +++- Sources/Workspace/FS/FileSystem.swift | 83 ++++- Sources/Workspace/FS/InMemoryFileSystem.swift | 39 +- Sources/Workspace/FS/LimitedFileSystem.swift | 20 +- Sources/Workspace/FS/LocalFileSystem.swift | 123 ++++++- Sources/Workspace/FS/MountedFileSystem.swift | 114 +++++- Sources/Workspace/FS/OverlayFileSystem.swift | 46 ++- Sources/Workspace/FS/SandboxFileSystem.swift | 20 +- .../FS/SecurityScopedFileSystem.swift | 19 +- Sources/Workspace/FilesystemObservation.swift | 252 +++++++++++++ Sources/Workspace/MetadataIndex.swift | 68 ++++ .../Workspace/Support/FileChangeStreams.swift | 95 ++++- .../Support/LocalAtomicTemporaryName.swift | 16 + Sources/Workspace/Support/Permissions.swift | 53 ++- Sources/Workspace/TransactionTree.swift | 304 ++++++++++++++++ Sources/Workspace/Workspace+Checkpoints.swift | 4 +- Sources/Workspace/Workspace+Edits.swift | 18 +- Sources/Workspace/Workspace+Internals.swift | 82 ++++- Sources/Workspace/Workspace+Merge.swift | 84 +++-- .../Workspace+TransactionStorage.swift | 103 ++++++ Sources/Workspace/Workspace.swift | 43 ++- Sources/Workspace/WorkspaceArchive.swift | 2 +- Sources/Workspace/WorkspaceChanges.swift | 4 + Sources/Workspace/WorkspaceCompaction.swift | 19 +- Sources/Workspace/WorkspaceEvent.swift | 7 +- Sources/Workspace/WorkspaceTransaction.swift | 246 +++++++------ Tests/WorkspaceTests/ChangeScalingTests.swift | 264 ++++++++++++++ .../WorkspaceTests/ChangeSoundnessTests.swift | 334 ++++++++++++++++++ Tests/WorkspaceTests/EditAtomicityTests.swift | 7 +- Tests/WorkspaceTests/EditPathTests.swift | 45 +++ .../FileSystemTraversalTests.swift | 36 +- .../LinuxFileChangeStreamTests.swift | 160 +++++++++ .../PathIdentityRegressionTests.swift | 38 ++ .../WorkspaceTests/ReviewRevisionTests.swift | 147 ++++++++ .../ScopedObservationTests.swift | 154 ++++++++ .../SearchEquivalenceTests.swift | 2 +- Tests/WorkspaceTests/TestSupport.swift | 39 +- Tests/WorkspaceTests/TransactionTests.swift | 18 +- 40 files changed, 2947 insertions(+), 250 deletions(-) create mode 100644 Sources/Workspace/FilesystemObservation.swift create mode 100644 Sources/Workspace/MetadataIndex.swift create mode 100644 Sources/Workspace/Support/LocalAtomicTemporaryName.swift create mode 100644 Sources/Workspace/TransactionTree.swift create mode 100644 Sources/Workspace/Workspace+TransactionStorage.swift create mode 100644 Tests/WorkspaceTests/ChangeScalingTests.swift create mode 100644 Tests/WorkspaceTests/ChangeSoundnessTests.swift create mode 100644 Tests/WorkspaceTests/ReviewRevisionTests.swift create mode 100644 Tests/WorkspaceTests/ScopedObservationTests.swift diff --git a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift index f82ba60..d5b8fdd 100644 --- a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift +++ b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift @@ -13,7 +13,8 @@ struct IOCounts: Codable, Equatable, Sendable { } /// A lock only protects counter updates; no lock is held across an await. -class CountingFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +class CountingFileSystem: FileSystem, TraversingFileSystem, ExactTimestampFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { + var hasExactContentTimestamps: Bool { get async { await base.supportsExactContentTimestamps } } let base: any FileSystem private let lock = NSLock() private var counts = IOCounts() @@ -156,6 +157,9 @@ class CountingFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvi } return try await resolver.resolvePotentialPath(path: path) } + func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + await base.previewBehavior(at: path) + } func capabilities() async -> FileSystemFeatures { call("capabilities") return await base.capabilities() @@ -166,6 +170,13 @@ class CountingFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvi final class ObservedCountingFileSystem: CountingFileSystem, FileSystemChangeSource, @unchecked Sendable { + func changeSignals() async throws -> AsyncStream { + call("changeSignals") + guard let source = base as? any FileSystemChangeSource else { + throw BenchmarkError.message("backend does not provide filesystem notifications") + } + return try await source.changeSignals() + } func changes() async throws -> AsyncStream { call("changes") guard let source = base as? any FileSystemChangeSource else { diff --git a/Benchmarks/WorkspaceBenchmarks/Scenarios.swift b/Benchmarks/WorkspaceBenchmarks/Scenarios.swift index 71926ce..59d931e 100644 --- a/Benchmarks/WorkspaceBenchmarks/Scenarios.swift +++ b/Benchmarks/WorkspaceBenchmarks/Scenarios.swift @@ -124,6 +124,10 @@ struct ScenarioRunner { // A no-op mutation loads AND prepares the store, without recording an edit. let priming = try await workspace.writeText(path, originalText) guard priming.isEmpty else { throw BenchmarkError.message("preparation recorded a mutation") } + // The steady-state transaction scenario excludes first-use base freezing. + // Ordinary checkpoints no longer opt an instance into transaction caching. + // transactionFirst measures that cold capture independently. + if operation == .transaction { try await workspace.beginTransaction().discard() } let checkpoint: Checkpoint? = operation.needsBaselineCheckpoint ? try await workspace.createCheckpoint() : nil @@ -284,9 +288,10 @@ struct ScenarioRunner { } private func assertObservation(_ counter: CountingFileSystem) throws { - guard counter.snapshot().filesRead >= Int64(fixture.fileCount) else { + let calls = counter.snapshot().calls + guard (calls["walk"] ?? 0) > 0 || (calls["listDirectory"] ?? 0) > 0 else { throw BenchmarkError.message( - "notification setup unavailable: events() did not capture the initial tree") + "notification setup unavailable: events() did not index the initial tree") } } diff --git a/README.md b/README.md index 608173a..c9c8a7b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,10 @@ print(result.changes.statistics.additions) `apply` is atomic by default. `.stopOnError` and `.continueAfterError` expose partial failures explicitly. Atomic recovery does not track hard-link aliases outside captured roots or roll back permission changes to symlink entries. +Edit previews use an isolated copy-on-write layer and the same affected-root journal as `apply`. +Their ChangeSets describe the edited paths, including link and permission changes. Preview does not +simulate external races or guarantee that later writes pass filesystem permissions or quota checks. + Direct conveniences—`writeText`, `writeData`, `appendText`, `createDirectory`, `remove`, `copy`, `move`, link creation, and permission changes—use the same edit pipeline. Text replacement is an edit and shares its file selection and pattern engine with search: @@ -280,7 +284,35 @@ let result = try await workspace.transaction { transaction in } ``` -Transactions are logically atomic within cooperative workspace use. They are not crash-safe against an external process racing the commit. +Transactions use a copy-on-write draft over a frozen base. The first transaction on an instance +reads the tree, including after reopening a Workspace. Memory persistence retains the initial bytes. +A cold directory-backed base retains up to 256 MiB of file contents in memory; files that do not fit +use private temporary blobs until a checkpoint supplies immutable content. Live transaction handles +own the cold base. Discarding or releasing the last handle releases its bytes; successful commit +releases the completed handle's base and re-seeds the parent from the checkpoint. Other active handles +retain their frozen contents. A later begin without a live cold base or checkpoint base reads anew. +Beginning or discarding a transaction adds no public checkpoint. Subsequent begins and commits walk +metadata and read changed or unverifiable files; previews and merges materialize affected paths. +Metadata capture uses bulk traversal when available and per-path enumeration otherwise, including +scoped subtree refreshes. Mounted traversal preserves virtual paths and shadowing; authorized +traversal retains per-entry permission checks. +Checkpoint persistence still requires the store's metadata and manifest work. Checkpoints prime a +transaction base only after that instance has begun a transaction. Directory commits replace the base +with the new revision index, without retaining changed-file contents or revision read caches. + +With `checkpointCapture: .trustMetadata`, transactions reuse contents when size and mtime match +and the previous mtime is more than two seconds older than the start of the previous metadata +capture. Identity must also match when both entries provide it. Missing mtimes and recent timestamps +require reads; InMemoryFileSystem's exact timestamps skip this racy window. Writes that preserve +safely old metadata can evade the check. Use `.verifyContents` to read and compare every file during +transaction begin and commit, and to read and hash every file during directory checkpoint capture. + +Transactions are logically atomic within cooperative workspace use. They are not crash-safe against +an external process racing the commit. In-process `Workspace.compact` protects revisions referenced +by transaction handles and other Workspace instances. This instance's reusable base does not override +retention: it survives dry runs and real compaction that retains its checkpoint, and is discarded +when that checkpoint is removed. Coordinate compaction across processes: the store protocol has no +cross-process revision lease. ## Events and History @@ -303,6 +335,37 @@ invalidations into structured `ChangeSet` values. External changes remain unreco history, matching the direct-filesystem contract above. Directory-backed checkpoint stores use the same push model for cross-instance checkpoint events; there is no periodic polling loop. +Filesystem observation keeps a metadata index and a bounded least-recently-used content cache. +`observation: .init(contentCacheBytes: 32 * 1024 * 1024)` configures the cache on `Workspace`; 32 MiB is +the default, zero disables it, and negative budgets are treated as zero. Initial observation reads +at most that budget, excluding files above the text-diff size limit. A change whose old or new bytes +are uncached has no text diff and reports `.observationCacheMiss` in `textDiffOmissions`. +`.sizeLimitExceeded` identifies files above the text-diff limit. Failed content reads report the +metadata change with `.contentReadFailed`, without blocking other changes. + +Full and recursive metadata rescans verify cached contents within the same two-second racy window +used by transactions, and whenever mtime is unavailable. Exact-timestamp backends skip that window. +Explicit path signals compare cached regular-file contents even when metadata matches. Rescans can +miss writes that preserve safely old metadata. Without cached contents, observation relies on +metadata and can report an identical rewrite as changed or miss a rewrite that preserves metadata. +Observation is not an atomic snapshot of external activity: a mutation between a Workspace write +and its metadata reconciliation can be absorbed into that write, particularly for uncached files +or structural changes. Coordinate concurrent writers when every intermediate change must be recorded. + +`FileSystemChangeSource.changeSignals()` emits `.paths([WorkspacePath: Scope])` in the source's +filesystem namespace. `.entry` checks one entry's contents and metadata; `.shallow` also checks a +directory's immediate listing without traversing existing child directories; `.recursive` checks +the whole subtree. Added, removed or replaced directories include their current or former descendants. +Sources must name deleted or replaced ancestors, use recursive scopes for directory renames, and emit +`.rescan` when coverage is uncertain. Coalescing keeps the strongest scope at each path without letting +a shallow ancestor suppress descendant signals. The default maps `changes()` Void signals to `.rescan`; +wrappers should forward richer signals when available. FSEvents and inotify provide scoped signals, +coalesced over 40 ms, with full metadata rescans for dropped or unknown events. Native watchers exclude +LocalFileSystem's atomic-replace temporary names in this process's private random namespace; similarly +named user files and temporary files from other processes remain visible. +Metadata failures retain their invalidations and retry with exponential backoff for at most three +attempts. After that, observation waits for a new native signal before resuming the pending work. + ## Path Identity A `WorkspacePath` is a normalized absolute path whose identity is the exact UTF-8 @@ -510,7 +573,7 @@ deadlines (30 minutes per child by default). | Scenario | What's timed | Setup excluded | Counter blind spots | | --- | --- | --- | --- | | `writeText`, `apply50`, `limitedWrite` | One recorded write, 50 edits, quota-limited write | No-op preparation, file reset | Checkpoint store; Limited metadata traversal is visible | -| `transaction` / `transactionFirst` | Begin, one draft write, commit | Preparation; baseline checkpoint for `transaction` only | Draft filesystem, checkpoint store | +| `transaction` / `transactionFirst` | Begin, one draft write, commit | Preparation; initial begin/discard and baseline checkpoint for steady-state `transaction` only | Draft filesystem, checkpoint store | | `preview` | One edit preview | Preparation | Draft filesystem | | `checkpointFirst` / `checkpointIncremental` | First capture / capture after one change | Preparation; baseline and change for incremental | Checkpoint store | | `diff`, `restore` | Checkpoint-to-current diff / restore after one change | Preparation, baseline, change | Checkpoint store | diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index ab69f85..9c7633c 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -210,8 +210,50 @@ public protocol FileSystem: AnyObject, Sendable { /// A filesystem that can push invalidations when its contents change outside Workspace APIs. public protocol FileSystemChangeSource: FileSystem { - /// Emits after one or more entries may have changed. Consumers should re-read the affected tree. + /// Emits an unscoped invalidation. Workspace consumes `changeSignals()`, whose + /// default implementation adapts this stream to `.rescan`. func changes() async throws -> AsyncStream + /// Emits scoped invalidations using the `FileSystemChangeSignal` contract. + func changeSignals() async throws -> AsyncStream +} + +/// Invalidates paths in the change source's own filesystem namespace. +/// Include deleted or replaced ancestors so their former descendants are checked. +/// Use `.rescan` when paths are unknown, events were dropped, or coverage is uncertain. +public enum FileSystemChangeSignal: Sendable, Equatable { + /// Invalidates the entire tree, including entries that no longer exist. + case rescan + /// Invalidates each path at its specified scope. Coalescing retains the strongest + /// scope for a path; a shallow ancestor does not subsume descendant signals. + case paths([WorkspacePath: Scope]) + + public enum Scope: Int, Sendable, Comparable { + /// Only this entry's contents or metadata changed. A removed or replaced + /// directory also invalidates its former descendants. + case entry + /// This entry and its immediate directory listing may have changed. Existing + /// child directories are not traversed; added or replaced subtrees are captured. + case shallow + /// Everything beneath this path is unknown, including former descendants. + /// Use for directory creation, removal or rename, and incomplete subtree coverage. + case recursive + + public static func < (lhs: Self, rhs: Self) -> Bool { lhs.rawValue < rhs.rawValue } + } +} + +extension FileSystemChangeSource { + /// Compatibility for sources that only provide unscoped invalidations. + public func changeSignals() async throws -> AsyncStream { + let stream = try await changes() + return AsyncStream { continuation in + let task = Task { + for await _ in stream where !Task.isCancelled { continuation.yield(.rescan) } + continuation.finish() + } + continuation.onTermination = { _ in task.cancel() } + } + } } /// Resolves links in the backend's namespace without requiring the target to exist. @@ -306,3 +348,42 @@ extension FileSystem { } typealias FileSystemCapabilities = FileSystemFeatures + +/// Backend differences relevant to copy-on-write edit previews. +package struct FileSystemPreviewSemantics: Sendable { + package var preservesOverwritePermissions: Bool + package var followsDirectorySymlinks: Bool + package var followsPermissionSymlinks: Bool + package var createsWriteParents: Bool + + package init(preservesOverwritePermissions: Bool = false, followsDirectorySymlinks: Bool = false, + followsPermissionSymlinks: Bool = false, createsWriteParents: Bool = false) { + self.preservesOverwritePermissions = preservesOverwritePermissions + self.followsDirectorySymlinks = followsDirectorySymlinks + self.followsPermissionSymlinks = followsPermissionSymlinks + self.createsWriteParents = createsWriteParents + } +} + +package protocol PreviewSemanticsFileSystem: FileSystem { + func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics +} + +extension FileSystem { + package func previewBehavior(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + if let source = self as? any PreviewSemanticsFileSystem { return await source.previewSemantics(at: path) } + return .init() + } +} + +/// Backends whose content changes always change mtime or identity, even within a clock tick. +/// Wrappers may forward this only when preserving those metadata guarantees. +package protocol ExactTimestampFileSystem: FileSystem { + var hasExactContentTimestamps: Bool { get async } +} + +extension FileSystem { + package var supportsExactContentTimestamps: Bool { + get async { await (self as? any ExactTimestampFileSystem)?.hasExactContentTimestamps ?? false } + } +} diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index af9da7e..1221666 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -13,7 +13,14 @@ import Glibc /// /// Actor isolation serializes access to the tree so the filesystem can be shared safely across /// concurrent tasks. -public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { +public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, ExactTimestampFileSystem { + package nonisolated var hasExactContentTimestamps: Bool { true } + private var lastTimestamp = Date().timeIntervalSinceReferenceDate + + private func nextTimestamp() -> Date { + lastTimestamp = max(Date().timeIntervalSinceReferenceDate, lastTimestamp.nextUp) + return Date(timeIntervalSinceReferenceDate: lastTimestamp) + } private final class Node { enum Kind { case file(Data) @@ -147,7 +154,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let existing = children[PathName(name)], case let .file(existingData) = existing.kind { existing.kind = .file(existingData + data) - existing.modificationDate = Date() + existing.modificationDate = nextTimestamp() children[PathName(name)] = existing } else { if let existing = children[PathName(name)] { unlink(existing) } @@ -156,7 +163,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { } parent.kind = .directory(children) - parent.modificationDate = Date() + parent.modificationDate = nextTimestamp() } /// See ``FileSystem/createDirectory(path:recursive:)``. @@ -190,7 +197,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { let directory = makeNode(kind: .directory([:]), permissions: 0o755) children[PathName(component)] = directory current.kind = .directory(children) - current.modificationDate = Date() + current.modificationDate = nextTimestamp() current = directory } } @@ -214,7 +221,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { parentChildren.removeValue(forKey: PathName(name)) unlink(entry) parent.kind = .directory(parentChildren) - parent.modificationDate = Date() + parent.modificationDate = nextTimestamp() } /// See ``FileSystem/move(from:to:)``. @@ -242,7 +249,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { children.removeValue(forKey: PathName(sourceName)) children[PathName(destinationName)] = sourceNode sourceParent.kind = .directory(children) - sourceParent.modificationDate = Date() + sourceParent.modificationDate = nextTimestamp() return } @@ -254,11 +261,11 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { var sourceChildren = try directoryChildren(of: sourceParent) sourceChildren.removeValue(forKey: PathName(sourceName)) sourceParent.kind = .directory(sourceChildren) - sourceParent.modificationDate = Date() + sourceParent.modificationDate = nextTimestamp() destinationChildren[PathName(destinationName)] = sourceNode destinationParent.kind = .directory(destinationChildren) - destinationParent.modificationDate = Date() + destinationParent.modificationDate = nextTimestamp() } /// See ``FileSystem/copy(from:to:recursive:)``. @@ -278,7 +285,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { destinationChildren[PathName(destinationName)] = clone(sourceNode) destinationParent.kind = .directory(destinationChildren) - destinationParent.modificationDate = Date() + destinationParent.modificationDate = nextTimestamp() } /// See ``FileSystem/createSymlink(path:target:)``. @@ -296,7 +303,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { children[PathName(name)] = makeNode(kind: .symlink(target), permissions: 0o777) parent.kind = .directory(children) - parent.modificationDate = Date() + parent.modificationDate = nextTimestamp() } /// See ``FileSystem/createHardLink(path:target:)``. @@ -319,7 +326,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { sourceNode.linkCount += 1 children[PathName(name)] = sourceNode parent.kind = .directory(children) - parent.modificationDate = Date() + parent.modificationDate = nextTimestamp() } /// See ``FileSystem/readSymlink(path:)``. @@ -337,7 +344,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { let node = try node(at: path, followFinalSymlink: false) node.permissions = Int(permissions.rawValue) - node.modificationDate = Date() + node.modificationDate = nextTimestamp() } /// See ``FileSystem/resolveRealPath(path:)``. @@ -381,10 +388,10 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { return matches.sorted() } - private func makeNode(kind: Node.Kind, permissions: Int, modificationDate: Date = Date()) -> Node { + private func makeNode(kind: Node.Kind, permissions: Int, modificationDate: Date? = nil) -> Node { let inode = nextInode nextInode += 1 - return Node(inode: inode, kind: kind, permissions: permissions, modificationDate: modificationDate) + return Node(inode: inode, kind: kind, permissions: permissions, modificationDate: modificationDate ?? nextTimestamp()) } private func clone(_ node: Node) -> Node { @@ -565,3 +572,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem { NSError(domain: NSPOSIXErrorDomain, code: Int(code)) } } + +extension InMemoryFileSystem: PreviewSemanticsFileSystem { + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { .init() } +} diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index 8ef7cde..8b46fcd 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -46,7 +46,7 @@ public enum FileSystemLimitError: Error, Sendable, Equatable, CustomStringConver /// Custom backends must follow standard mutation semantics: replacement changes one entry; /// append updates aliases; parents may be created. A composition cycle (for example a Mounted /// base that mounts this same LimitedFileSystem) deadlocks while the gate is held. -public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem { +public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem { private let base: any FileSystem private let pathResolver: FileSystemPathResolver public let limits: FileSystemLimits @@ -77,6 +77,10 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PotentialPathR public func readFileChunks(path: WorkspacePath, chunkSize: Int) async throws -> AsyncThrowingStream { try await base.readFileChunks(path: path, chunkSize: chunkSize) } + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + await base.previewBehavior(at: path) + } + public func capabilities() async -> FileSystemFeatures { await base.capabilities() } public func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { @@ -424,3 +428,17 @@ struct FileSystemPathResolver: Sendable { && [NSFileNoSuchFileError, NSFileReadNoSuchFileError].contains(error.code)) } } + + +extension LimitedFileSystem: FileSystemChangeSource { + public func changes() async throws -> AsyncStream { + unscopedChanges(try await changeSignals()) + } + + public func changeSignals() async throws -> AsyncStream { + guard let source = base as? any FileSystemChangeSource else { + throw WorkspaceError.unsupported("filesystem change notifications are not supported") + } + return try await source.changeSignals() + } +} diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 5f8117e..f8e6078 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -18,7 +18,7 @@ private typealias LocalStat = Glibc.stat /// This is a safety layer, not a sandbox: a host process can relocate an already-open directory /// outside the root, change hard-linked data, or mutate entries during recursive operations. /// Watch subscriptions still use the configured URL and are not descriptor-pinned. -public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem, TraversingFileSystem, @unchecked Sendable { +public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// Returns zero on success or a captured POSIX errno on failure. Injection is per-instance /// so unsupported filesystem behavior can be tested without mounts or global syscall hooks. typealias ExclusiveRename = @Sendable (Int32, String, Int32, String) -> Int32 @@ -824,7 +824,7 @@ public final class LocalFileSystem: FileSystem, PotentialPathResolvingFileSystem private func replace(_ entry: Entry, body: (Descriptor) throws -> Void) throws { try entry.requireVerified() - let temporary = ".workspace-" + UUID().uuidString + let temporary = LocalAtomicTemporaryName.make() let temporaryEntry = try verifiedEntry(Entry(parent: entry.parent, name: temporary, path: entry.path), creating: true) let fd = try openEntry(temporaryEntry, flags: O_WRONLY | O_CREAT | O_EXCL) defer { @@ -1086,20 +1086,94 @@ extension LocalFileSystem: FileSystemChangeSource { let root = try requireConfiguration().rootURL return try FSEventsChangeStream.observe(root) } + + public func changeSignals() async throws -> AsyncStream { + let configuration = try requireConfiguration() + return try FSEventsChangeStream.observeSignals(configuration.rootURL, resolvedRootPath: configuration.resolvedRootPath) + } } enum FSEventsChangeStream { + // FileEvents supplies item flags; directory metadata/listing changes do not + // invalidate descendants unless structural or loss/coalescing flags say so. + static func scope(for flags: FSEventStreamEventFlags) -> FileSystemChangeSignal.Scope? { + let unknown = UInt32(kFSEventStreamEventFlagUserDropped | kFSEventStreamEventFlagKernelDropped + | kFSEventStreamEventFlagRootChanged | kFSEventStreamEventFlagEventIdsWrapped + | kFSEventStreamEventFlagMount | kFSEventStreamEventFlagUnmount) + if flags & unknown != 0 { return nil } + if flags & UInt32(kFSEventStreamEventFlagMustScanSubDirs) != 0 { return .recursive } + if flags & UInt32(kFSEventStreamEventFlagItemIsDir) != 0 { + let structural = UInt32(kFSEventStreamEventFlagItemCreated | kFSEventStreamEventFlagItemRemoved + | kFSEventStreamEventFlagItemRenamed) + return flags & structural != 0 ? .recursive : .shallow + } + if flags & UInt32(kFSEventStreamEventFlagItemIsFile | kFSEventStreamEventFlagItemIsSymlink) != 0 { return .entry } + return nil // No item type: do not assume directory-level coverage is complete. + } + + /// Stream-local identity distinguishes late item events for the existing root + /// from replacement of the watched directory. Native names need no spelling lookup. + struct EventMapper { + let root: String + private let initialIdentity: FileIdentity? + #if DEBUG + private let trace = ProcessInfo.processInfo.environment["WORKSPACE_FSEVENTS_TRACE"] == "1" + #endif + + init(root: String) { + self.root = root + initialIdentity = Self.identity(at: root) + } + + private static func identity(at path: String) -> FileIdentity? { + var info = LocalStat() + guard lstat(path, &info) == 0, info.st_mode & S_IFMT == S_IFDIR else { return nil } + return FileIdentity(device: UInt64(truncatingIfNeeded: info.st_dev), inode: UInt64(info.st_ino)) + } + + func signal(for events: [(path: String, flags: FSEventStreamEventFlags)]) -> FileSystemChangeSignal? { + let currentIdentity = Self.identity(at: root) + #if DEBUG + if trace { + for event in events { + let message = "Workspace FSEvents root=\(String(reflecting: root)) path=\(String(reflecting: event.path)) flags=0x\(String(event.flags, radix: 16)) initialIdentity=\(String(describing: initialIdentity)) currentIdentity=\(String(describing: currentIdentity))\n" + FileHandle.standardError.write(Data(message.utf8)) + } + } + #endif + guard let initialIdentity, currentIdentity == initialIdentity else { return .rescan } + var changed: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + for event in events { + guard var scope = scope(for: event.flags), + let path = nativeChangePath(event.path, relativeTo: root) else { return .rescan } + if path.isRoot { + // Coalesced subtree coverage is still mandatory, even for an + // unchanged root. Ordinary root item events only affect its listing. + if event.flags & UInt32(kFSEventStreamEventFlagMustScanSubDirs) != 0 { return .rescan } + scope = .shallow + } + if LocalAtomicTemporaryName.matches(path.name) { continue } + changed[path] = max(changed[path] ?? .entry, scope) + } + return changed.isEmpty ? nil : .paths(changed) + } + } + private final class Box: @unchecked Sendable { - let continuation: AsyncStream.Continuation - init(_ continuation: AsyncStream.Continuation) { self.continuation = continuation } + let continuation: AsyncStream.Continuation + let mapper: EventMapper + init(_ continuation: AsyncStream.Continuation, resolvedRootPath: String) { + self.continuation = continuation + self.mapper = EventMapper(root: resolvedRootPath) + } } private final class Observation: @unchecked Sendable { private let stream: FSEventStreamRef private let box: Unmanaged - init(root: URL, continuation: AsyncStream.Continuation) throws { - box = .passRetained(Box(continuation)) + init(root: URL, resolvedRootPath: String, continuation: AsyncStream.Continuation) throws { + box = .passRetained(Box(continuation, resolvedRootPath: resolvedRootPath)) var context = FSEventStreamContext( version: 0, info: box.toOpaque(), @@ -1107,12 +1181,17 @@ enum FSEventsChangeStream { release: nil, copyDescription: nil ) - let callback: FSEventStreamCallback = { _, info, _, _, _, _ in + let callback: FSEventStreamCallback = { _, info, count, paths, flags, _ in guard let info else { return } - Unmanaged.fromOpaque(info).takeUnretainedValue().continuation.yield() + let box = Unmanaged.fromOpaque(info).takeUnretainedValue() + let names = unsafeBitCast(paths, to: NSArray.self) as! [String] + let events = (0.. AsyncStream { + let signals = try observeSignals(root) + return unscopedChanges(signals) + } + + static func observeSignals(_ root: URL, resolvedRootPath: String? = nil) throws -> AsyncStream { + let canonical: String + if let resolvedRootPath { canonical = resolvedRootPath } + else if let path = realpath(root.path, nil) { + canonical = String(cString: path) + free(path) + } else { canonical = root.path } var failure: Error? - let stream = AsyncStream { continuation in + let stream = AsyncStream { continuation in do { - let observation = try Observation(root: root, continuation: continuation) + let observation = try Observation(root: root, resolvedRootPath: canonical, continuation: continuation) continuation.onTermination = { _ in observation.cancel() } } catch { failure = error @@ -1165,5 +1255,16 @@ extension LocalFileSystem: FileSystemChangeSource { public func changes() async throws -> AsyncStream { try DirectoryChangeStream.observe(requireConfiguration().rootURL, recursive: true) } + + public func changeSignals() async throws -> AsyncStream { + try DirectoryChangeStream.observeSignals(requireConfiguration().rootURL, recursive: true) + } } #endif + +extension LocalFileSystem { + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + .init(preservesOverwritePermissions: true, followsDirectorySymlinks: true, + followsPermissionSymlinks: true, createsWriteParents: true) + } +} diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index 9246a09..7165535 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -10,7 +10,7 @@ import Glibc /// /// Each mount is exposed at a virtual path, allowing callers to present a single workspace assembled /// from multiple independent roots. -public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// A mounted filesystem and the virtual path where it appears. public struct Mount: Sendable { /// The normalized virtual mount point. @@ -37,6 +37,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst private let base: any FileSystem private let stateLock = NSLock() private var mounts: [Mount] + private var mountObservers: [UUID: AsyncStream<[Mount]>.Continuation] = [:] private func withLock(_ body: () throws -> R) rethrows -> R { stateLock.lock() @@ -63,6 +64,7 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst let mount = Mount(mountPoint: mountPoint, fileSystem: fileSystem) mounts.append(mount) mounts.sort { $0.mountPoint.components.count > $1.mountPoint.components.count } + for observer in mountObservers.values { observer.yield(mounts) } } } @@ -75,6 +77,57 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst mount(WorkspacePath(normalizing: mountPoint), fileSystem: filesystem) } + package var supportsScopedWalk: Bool { + base.supportsScopedWalk || mountsSnapshot().contains { $0.fileSystem.supportsScopedWalk } + } + + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + let mounts = mountsSnapshot() + // An entire subtree in one backend can use its descriptor-scoped walk. + // At mount junctions, virtual listings retain shadowing and synthetic parents. + if !mounts.contains(where: { $0.mountPoint != path && $0.mountPoint.isWithin(path) }) { + let resolved = resolveWritable(path: path, mounts: mounts) + var visited = false + do { + try await resolved.filesystem.walk(at: resolved.relativePath, + skippingUnreadableDirectories: skippingUnreadableDirectories) { info, read, readLink in + visited = true + let virtual = resolved.mountPoint.appending(info.path.components.joined(separator: "/")) + try await withoutActuallyEscaping(read) { scopedRead in + try await withoutActuallyEscaping(readLink) { scopedLink in + try await visit(Self.virtualInfo(info, at: virtual), + { try await scopedRead($0) }, { try await scopedLink() }) + } + } + } + } catch { + // A backend root may be a skippable child in our virtual namespace. + guard !visited, resolved.relativePath.isRoot, skippingUnreadableDirectories, + isSkippableWalkError(error, at: path) else { throw error } + try await walkUsingPaths(at: path, skippingUnreadableDirectories: true, visit: visit) + } + return + } + guard let info = try await Snapshot.infoIfPresent(at: path, on: self) else { return } + if info.kind == .directory { + let listing: [DirectoryEntry] + do { listing = try await listDirectory(path: path) } + catch { + guard skippingUnreadableDirectories, isSkippableWalkError(error, at: path) else { throw error } + try await visit(info, { _ in throw WorkspaceError.unsupported("cannot read a directory") }, + { throw WorkspaceError.unsupported("not a symlink") }) + return + } + for child in listing.sorted(by: { PathName(walkSortKey($0)) < PathName(walkSortKey($1)) }) { + try await walk(at: child.info.path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + } + try await visit(info, { length in + try await self.readFile(path: path, offset: 0, length: length) + }, { try await self.readSymlink(path: path) }) + } + /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { let mounts = mountsSnapshot() @@ -169,6 +222,11 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst try await resolved.filesystem.createFile(path: resolved.relativePath, data: data) } + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + let resolved = resolveWritable(path: path, mounts: mountsSnapshot()) + return await resolved.filesystem.previewBehavior(at: resolved.relativePath) + } + /// See ``FileSystem/capabilities()``. Reports the intersection of the base filesystem's and /// every mount's capabilities, since a path may resolve to any of them. public func capabilities() async -> FileSystemFeatures { @@ -433,3 +491,57 @@ public final class MountedFileSystem: FileSystem, PotentialPathResolvingFileSyst return nil } } + +extension MountedFileSystem: FileSystemChangeSource { + public func changes() async throws -> AsyncStream { + unscopedChanges(try await changeSignals()) + } + + public func changeSignals() async throws -> AsyncStream { + let id = UUID() + let (topology, updates) = AsyncStream<[Mount]>.makeStream(bufferingPolicy: .bufferingNewest(1)) + withLock { + mountObservers[id] = updates + updates.yield(mounts) + } + return AsyncStream { continuation in + let task = Task { + var subscriptions: [Task] = [] + defer { subscriptions.forEach { $0.cancel() } } + for await mounts in topology where !Task.isCancelled { + subscriptions.forEach { $0.cancel() } + subscriptions.removeAll() + let sources = [Mount(mountPoint: .root, fileSystem: self.base)] + mounts + for mount in sources { + guard let source = mount.fileSystem as? any FileSystemChangeSource, + let signals = try? await source.changeSignals() else { continue } + subscriptions.append(Task { + for await signal in signals where !Task.isCancelled { + switch signal { + case .rescan: + continuation.yield(.paths([mount.mountPoint: .recursive])) + case let .paths(paths): + var visible: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + for (path, scope) in paths { + let virtual = mount.mountPoint.appending(path.components.joined(separator: "/")) + let owner = self.resolveWritable(path: virtual, mounts: mounts) + guard owner.mountPoint == mount.mountPoint else { continue } + visible[virtual] = scope + } + if !visible.isEmpty { continuation.yield(.paths(visible)) } + } + } + }) + } + // Reconcile changes during subscription setup and any new mount. + continuation.yield(.rescan) + } + continuation.finish() + } + continuation.onTermination = { _ in + task.cancel() + self.withLock { self.mountObservers.removeValue(forKey: id)?.finish() } + } + } + } +} diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index 286554f..ea3ae28 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -19,7 +19,7 @@ import Glibc /// /// `reload()` (or `configure(rootDirectory:)`) discards the upper layer and whiteouts, so the /// overlay reflects the source directory's current state again. -public actor OverlayFileSystem: FileSystem { +public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { private let fileManager: FileManager private var upper: InMemoryFileSystem private var lower: (any FileSystem)? @@ -28,6 +28,8 @@ public actor OverlayFileSystem: FileSystem { /// merged view. Creating a new entry at a cut path shadows it in the upper layer; the cut /// keeps hiding the original lower content. private var cuts: Set = [] + private var rootPermissionsChanged = false + private var previewSource: (any FileSystem)? /// Creates an unconfigured overlay filesystem. init(fileManager: FileManager = .default) { @@ -43,6 +45,26 @@ public actor OverlayFileSystem: FileSystem { rootURL = nil } + init(previewOf base: any FileSystem) { + fileManager = .default + upper = InMemoryFileSystem() + lower = base + previewSource = base + } + + init(over base: any FileSystem, semanticsOf source: any FileSystem) { + fileManager = .default + upper = InMemoryFileSystem() + lower = base + previewSource = source + } + + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + if let previewSource { return await previewSource.previewBehavior(at: path) } + return .init(preservesOverwritePermissions: true, + followsDirectorySymlinks: true, followsPermissionSymlinks: true) + } + /// Creates and configures an overlay for `rootDirectory`. init(rootDirectory: URL, fileManager: FileManager = .default) async throws { self.fileManager = fileManager @@ -75,6 +97,7 @@ public actor OverlayFileSystem: FileSystem { public func discardChanges() async throws { upper = InMemoryFileSystem() cuts = [] + rootPermissionsChanged = false if rootURL != nil { try resetLayers() } @@ -83,6 +106,7 @@ public actor OverlayFileSystem: FileSystem { private func resetLayers() throws { upper = InMemoryFileSystem() cuts = [] + rootPermissionsChanged = false guard let rootURL else { lower = nil return @@ -104,6 +128,9 @@ public actor OverlayFileSystem: FileSystem { throw posixError(ENOENT) } var adjusted = info + if path.isRoot, !rootPermissionsChanged, let lower { + adjusted.permissions = try await lower.stat(path: .root).permissions + } adjusted.path = path return adjusted } @@ -219,9 +246,12 @@ public actor OverlayFileSystem: FileSystem { guard !path.isRoot else { throw posixError(EISDIR) } + let semantics = await previewSemantics(at: path) + if semantics.createsWriteParents { try await createDirectory(path: path.dirname, recursive: true) } let target = try await resolveMergedPath(path) - if await upperEntry(target) != nil { + if let info = await upperEntry(target) { try await upper.writeFile(path: target, data: data, append: append) + if semantics.preservesOverwritePermissions { try await upper.setPermissions(path: target, permissions: info.permissions) } return } if let info = await lowerVisibleEntry(target), let lower { @@ -236,7 +266,7 @@ public actor OverlayFileSystem: FileSystem { contents = data } try await upper.writeFile(path: target, data: contents, append: false) - try await upper.setPermissions(path: target, permissions: info.permissions) + try await upper.setPermissions(path: target, permissions: !semantics.preservesOverwritePermissions && !append ? .defaultFile : info.permissions) return } try await ensureUpperDirectories(for: target) @@ -255,6 +285,13 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/createDirectory(path:recursive:)``. public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { + if !(await previewSemantics(at: path)).followsDirectorySymlinks { + var current = WorkspacePath.root + for component in path.components { + current = current.appending(component) + if let info = await mergedEntry(current), info.kind != .directory { throw posixError(ENOTDIR) } + } + } let path = try await resolveMergedPath(path, allowMissingAncestors: recursive) if path.isRoot { return @@ -394,7 +431,7 @@ public actor OverlayFileSystem: FileSystem { /// See ``FileSystem/setPermissions(path:permissions:)``. public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { - let path = try await resolveMergedPath(path) + let path = try await resolveMergedPath(path, followFinal: previewSemantics(at: path).followsPermissionSymlinks) if await upperEntry(path) == nil { guard let info = await lowerVisibleEntry(path) else { throw posixError(ENOENT) @@ -407,6 +444,7 @@ public actor OverlayFileSystem: FileSystem { } } try await upper.setPermissions(path: path, permissions: permissions) + if path.isRoot { rootPermissionsChanged = true } } // MARK: - Merged view diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index 7abb16a..101a7bc 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -1,7 +1,7 @@ import Foundation /// A disk-backed filesystem rooted in a standard app sandbox location. -public final class SandboxFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem, Sendable { +public final class SandboxFileSystem: FileSystem, TraversingFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, Sendable { /// Supported strategies for choosing the sandbox root. public enum Root: Sendable { /// The user's documents directory. @@ -63,6 +63,10 @@ public final class SandboxFileSystem: FileSystem, TraversingFileSystem, Potentia try await backing.createFile(path: path, data: data) } + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + await backing.previewBehavior(at: path) + } + /// See ``FileSystem/capabilities()``. public func capabilities() async -> FileSystemFeatures { await backing.capabilities() @@ -167,3 +171,17 @@ public final class SandboxFileSystem: FileSystem, TraversingFileSystem, Potentia } typealias SandboxFilesystem = SandboxFileSystem + + +extension SandboxFileSystem: FileSystemChangeSource { + public func changes() async throws -> AsyncStream { + unscopedChanges(try await changeSignals()) + } + + public func changeSignals() async throws -> AsyncStream { + guard let source = (backing as any FileSystem) as? any FileSystemChangeSource else { + throw WorkspaceError.unsupported("filesystem change notifications are not supported") + } + return try await source.changeSignals() + } +} diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index 9fe13c9..0f1feb4 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -5,7 +5,7 @@ import Foundation /// /// This adapter is intended for iOS and macOS workflows where filesystem access must be granted through /// a bookmark or document picker URL. -public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { /// The permitted mutability for the scoped filesystem. public enum AccessMode: Sendable { /// Only read operations are allowed. @@ -167,6 +167,10 @@ public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, P try await backing.createFile(path: path, data: data) } + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + await backing.previewBehavior(at: path) + } + /// See ``FileSystem/capabilities()``. public func capabilities() async -> FileSystemFeatures { await backing.capabilities() @@ -293,4 +297,17 @@ public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, P } typealias SecurityScopedFilesystem = SecurityScopedFileSystem + +extension SecurityScopedFileSystem: FileSystemChangeSource { + public func changes() async throws -> AsyncStream { + unscopedChanges(try await changeSignals()) + } + + public func changeSignals() async throws -> AsyncStream { + guard let source = (backing as any FileSystem) as? any FileSystemChangeSource else { + throw WorkspaceError.unsupported("filesystem change notifications are not supported") + } + return try await source.changeSignals() + } +} #endif diff --git a/Sources/Workspace/FilesystemObservation.swift b/Sources/Workspace/FilesystemObservation.swift new file mode 100644 index 0000000..3862caa --- /dev/null +++ b/Sources/Workspace/FilesystemObservation.swift @@ -0,0 +1,252 @@ +import Foundation + +struct FilesystemObservation: Sendable { + var metadata: MetadataIndex + var cache: [WorkspacePath: Data] = [:] + var cacheBytes = 0 + private struct Recency: Sendable { var older: WorkspacePath?; var newer: WorkspacePath? } + private var recency: [WorkspacePath: Recency] = [:] + private var oldest: WorkspacePath? + private var newest: WorkspacePath? + var aliases: [FileIdentity: Set] = [:] + private var children: [WorkspacePath: Set] = [:] + let budget: Int + + static func start(on fs: any FileSystem, budget: Int) async throws -> Self { + let metadata = try await MetadataIndex.capture(from: fs) + var result = Self(metadata: metadata, budget: budget) + for (path, entry) in metadata.entries { + if path != .root { result.children[path.parent, default: []].insert(path) } + if entry.info.kind == .file, let identity = entry.info.identity { + result.aliases[identity, default: []].insert(path) + } + } + guard budget > 0 else { return result } + // Warm only the bounded text cache. Large files never enter it. + for path in metadata.entries.keys.sorted() { + let info = metadata.entries[path]!.info + if info.kind == .file && info.size <= UInt64(DiffOptions.default.maxTextBytes!) + && info.size <= UInt64(budget - result.cacheBytes) { + if let data = try? await fs.readFile(path: path) { result.insert(data, at: path) } + } + } + return result + } + + private mutating func unlink(_ path: WorkspacePath) { + guard let links = recency.removeValue(forKey: path) else { return } + if let older = links.older { recency[older]?.newer = links.newer } else { oldest = links.newer } + if let newer = links.newer { recency[newer]?.older = links.older } else { newest = links.older } + } + + private mutating func touch(_ path: WorkspacePath) { + guard cache[path] != nil else { return } + unlink(path) + recency[path] = Recency(older: newest, newer: nil) + if let newest { recency[newest]?.newer = path } else { oldest = path } + newest = path + } + + private mutating func removeCached(_ path: WorkspacePath) { + if let old = cache.removeValue(forKey: path) { cacheBytes -= old.count } + unlink(path) + } + + mutating func insert(_ data: Data, at path: WorkspacePath) { + removeCached(path) + guard budget > 0, data.count <= budget, data.count <= DiffOptions.default.maxTextBytes! else { return } + while cacheBytes + data.count > budget, let victim = oldest { removeCached(victim) } + cache[path] = data + cacheBytes += data.count + touch(path) + } + + mutating func replace(root: WorkspacePath, metadata replacement: MetadataIndex) { + let removed = subtree(root) + for path in removed { + children.removeValue(forKey: path) + children[path.parent]?.remove(path) + if let old = metadata.entries.removeValue(forKey: path), let identity = old.info.identity { + aliases[identity]?.remove(path) + if aliases[identity]?.isEmpty == true { aliases.removeValue(forKey: identity) } + } + removeCached(path) + } + metadata.entries.merge(replacement.entries) { _, new in new } + for (path, entry) in replacement.entries { + if path != .root { children[path.parent, default: []].insert(path) } + if entry.info.kind == .file, let identity = entry.info.identity { aliases[identity, default: []].insert(path) } + } + } + + mutating func remember(_ entry: Snapshot.Entry) { + switch entry { + case let .file(file): insert(file.data, at: file.path) + case let .directory(directory): directory.children.forEach { remember($0) } + default: break + } + } + + private func subtree(_ root: WorkspacePath) -> [WorkspacePath] { + var paths = [root], cursor = 0 + while cursor < paths.count { + paths.append(contentsOf: children[paths[cursor]] ?? []) + cursor += 1 + } + return paths + } + + mutating func refresh(on fs: any FileSystem, signal: FileSystemChangeSignal) async throws -> ChangeSet { + var before = MetadataIndex(), next = MetadataIndex() + before.capturedAt = metadata.capturedAt + before.exactTimestamps = metadata.exactTimestamps + var verifyContents: Set = [] + var checkRacyContents: Set = [] + switch signal { + case .rescan: + before = metadata + next = try await MetadataIndex.capture(from: fs) + checkRacyContents = Set(next.entries.keys) + case let .paths(paths): + var affected = paths + for path in paths.keys { + if let identity = metadata.entries[path]?.info.identity, let shared = aliases[identity] { + for alias in shared { affected[alias] = max(affected[alias] ?? .entry, .entry) } + } + } + var recursiveRoots: [WorkspacePath] = [] + func captureRecursive(_ root: WorkspacePath) async throws { + for path in subtree(root) { before.entries[path] = metadata.entries[path] } + let captured = try await MetadataIndex.capture(from: fs, at: root) + next.entries.merge(captured.entries) { _, new in new } + checkRacyContents.formUnion(captured.entries.keys) + recursiveRoots.append(root) + } + func captureEntry(_ info: FileInfo) async throws { + before.entries[info.path] = metadata.entries[info.path] + let target = info.kind == .symlink ? try await fs.readSymlink(path: info.path) : nil + next.entries[info.path] = .init(info: info, target: target) + } + for path in affected.keys.sorted() { + // Preserve explicit file verification even beneath a shallow/recursive ancestor. + verifyContents.insert(path) + if recursiveRoots.contains(where: { Snapshot.contains($0, path) }) { continue } + let scope = affected[path]! + let old = metadata.entries[path] + let info = try await Snapshot.infoIfPresent(at: path, on: fs) + let replacedDirectory = old?.info.kind == .directory && + (info?.kind != .directory || old?.info.identity != info?.identity) + if scope == .recursive || replacedDirectory || (info?.kind == .directory && old?.info.kind != .directory) { + try await captureRecursive(path) + continue + } + before.entries[path] = old + guard let info else { continue } + try await captureEntry(info) + guard scope == .shallow, info.kind == .directory else { continue } + let listing = try await fs.listDirectory(path: path) + let live = Set(listing.map { $0.info.path }) + for removed in (children[path] ?? []).subtracting(live) { + for descendant in subtree(removed) { before.entries[descendant] = metadata.entries[descendant] } + } + for child in listing { + let prior = metadata.entries[child.info.path] + if (child.info.kind == .directory || prior?.info.kind == .directory) && + (prior?.info.kind != child.info.kind || prior?.info.identity != child.info.identity) { + try await captureRecursive(child.info.path) + } else { + try await captureEntry(child.info) + } + } + } + } + var changes: [FileChange] = [] + var omissions: [TextDiffOmission] = [] + var invalidated: Set = [] + var used: [WorkspacePath] = [] + var admitted: [WorkspacePath: Data] = [:] + var admittedBytes = 0 + var admittedOrder: [WorkspacePath] = [] + var admittedStart = 0 + for path in Set(before.entries.keys).union(next.entries.keys).sorted() { + let old = before.entries[path], new = next.entries[path] + let requiresContentCheck = new?.info.kind == .file && cache[path] != nil && ( + verifyContents.contains(path) + || (checkRacyContents.contains(path) && old.map { new?.canReuseContents($0, capturedAt: before.capturedAt, exactTimestamps: before.exactTimestamps) != true } != false)) + guard old != new || requiresContentCheck else { continue } + // mtime/identity changes on directories alone are not workspace changes. + if let old, let new, old.info.kind == .directory, new.info.kind == .directory, + old.info.permissions == new.info.permissions { continue } + let oldData = cache[path] + if oldData != nil { used.append(path) } + var newData: Data? + var contentReadFailed = false + if budget > 0, let new, new.info.kind == .file, new.info.size <= UInt64(DiffOptions.default.maxTextBytes!), + new.info.size <= UInt64(budget) { + do { newData = try await fs.readFile(path: path) } + catch { contentReadFailed = true } + } + if let oldData, let newData, old?.info.kind == .file, new?.info.kind == .file, + oldData == newData, old?.info.permissions == new?.info.permissions { continue } + + func entry(_ value: MetadataIndex.Entry?, _ data: Data?) -> Snapshot.Entry { + guard let value else { return .missing(.init(path: path)) } + switch value.info.kind { + case .file: return .file(.init(path: path, data: data ?? Data(), permissions: value.info.permissions)) + case .directory: return .directory(.init(path: path, permissions: value.info.permissions, children: [])) + case .symlink: return .symlink(.init(path: path, target: value.target!, permissions: value.info.permissions)) + } + } + let missingContent = (old?.info.kind == .file && oldData == nil) || (new?.info.kind == .file && newData == nil) + if missingContent { + if let old, let new, old.info.kind == new.info.kind { + changes.append(.init(path: path, kind: new.info.kind, effect: .modified)) + } else { + if let old { changes.append(.init(path: path, kind: old.info.kind, effect: .deleted)) } + if let new { changes.append(.init(path: path, kind: new.info.kind, effect: .created)) } + } + let oversized = max(old?.info.size ?? 0, new?.info.size ?? 0) > UInt64(DiffOptions.default.maxTextBytes!) + omissions.append(.init(path: path, reason: contentReadFailed ? .contentReadFailed : (oversized ? .sizeLimitExceeded : .observationCacheMiss))) + } else { + let diff = ChangeSet.compare(before: Snapshot(rootPath: path, entry: entry(old, oldData)), + after: Snapshot(rootPath: path, entry: entry(new, newData)), + maxTextBytes: DiffOptions.default.maxTextBytes) + changes += diff.changes + omissions += diff.textDiffOmissions + } + invalidated.insert(path) + if let newData { + while admittedBytes + newData.count > budget, admittedStart < admittedOrder.count { + let victim = admittedOrder[admittedStart] + admittedStart += 1 + admittedBytes -= admitted.removeValue(forKey: victim)!.count + } + admittedOrder.append(path) + admitted[path] = newData + admittedBytes += newData.count + } + } + // Commit cache and metadata together only after every changed-file read succeeds. + for path in used { touch(path) } + for path in invalidated { + removeCached(path) + } + for path in admittedOrder.dropFirst(admittedStart) { insert(admitted[path]!, at: path) } + // Update index membership without invalidating the freshly updated cache. + for path in before.entries.keys { + children[path.parent]?.remove(path) + if next.entries[path]?.info.kind != .directory { children.removeValue(forKey: path) } + if let old = metadata.entries.removeValue(forKey: path), let identity = old.info.identity { + aliases[identity]?.remove(path) + if aliases[identity]?.isEmpty == true { aliases.removeValue(forKey: identity) } + } + } + if case .rescan = signal { metadata.capturedAt = next.capturedAt } + metadata.entries.merge(next.entries) { _, new in new } + for (path, entry) in next.entries { + if path != .root { children[path.parent, default: []].insert(path) } + if entry.info.kind == .file, let identity = entry.info.identity { aliases[identity, default: []].insert(path) } + } + return ChangeSet(changes: changes, textDiffOmissions: omissions) + } +} diff --git a/Sources/Workspace/MetadataIndex.swift b/Sources/Workspace/MetadataIndex.swift new file mode 100644 index 0000000..fcb7dad --- /dev/null +++ b/Sources/Workspace/MetadataIndex.swift @@ -0,0 +1,68 @@ +import Foundation + +/// An lstat index: directory timestamps are hints, never logical changes. +struct MetadataIndex: Sendable { + struct Entry: Sendable, Equatable { + var info: FileInfo + var target: String? + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.info.kind == rhs.info.kind && lhs.info.size == rhs.info.size + && lhs.info.modificationDate == rhs.info.modificationDate + && lhs.info.identity == rhs.info.identity && lhs.info.permissions == rhs.info.permissions + && lhs.target.map(PathName.init) == rhs.target.map(PathName.init) + } + + func canReuseContents(_ old: Self, capturedAt: Date, exactTimestamps: Bool = false) -> Bool { + info.kind == .file && old.info.kind == .file + && old.info.modificationDate.map { exactTimestamps || $0.addingTimeInterval(2) < capturedAt } == true + && info.size == old.info.size && info.modificationDate == old.info.modificationDate + && (info.identity == nil || old.info.identity == nil || info.identity == old.info.identity) + } + } + var entries: [WorkspacePath: Entry] = [:] + // Set before enumeration; never advance while reusing entries from an older walk. + var capturedAt = Date() + var exactTimestamps = false + + static func capture(from fs: any FileSystem, at root: WorkspacePath = .root) async throws -> Self { + var result = Self() + result.exactTimestamps = await fs.supportsExactContentTimestamps + // A scoped leaf reconciliation needs only one stat, not another walk. + // Whole-tree capture starts directly at the traversal seam. + let rootInfo: FileInfo? + if root.isRoot { + rootInfo = nil + } else { + guard let info = try await Snapshot.infoIfPresent(at: root, on: fs) else { return result } + rootInfo = info + if info.kind != .directory { + let target = info.kind == .symlink ? try await fs.readSymlink(path: root) : nil + result.entries[root] = Entry(info: info, target: target) + return result + } + } + if fs.supportsScopedWalk { + try await fs.walk(at: root) { info, _, readLink in + let target = info.kind == .symlink ? try await readLink() : nil + result.entries[info.path] = Entry(info: info, target: target) + } + return result + } + func visit(_ info: FileInfo) async throws { + let target = info.kind == .symlink ? try await fs.readSymlink(path: info.path) : nil + result.entries[info.path] = Entry(info: info, target: target) + if info.kind == .directory { + for child in try await fs.listDirectory(path: info.path).sorted(by: { PathName($0.name) < PathName($1.name) }) { + try await visit(child.info) + } + } + } + if let rootInfo { + try await visit(rootInfo) + } else if let info = try await Snapshot.infoIfPresent(at: root, on: fs) { + try await visit(info) + } + return result + } +} diff --git a/Sources/Workspace/Support/FileChangeStreams.swift b/Sources/Workspace/Support/FileChangeStreams.swift index 5e67c74..ee959e5 100644 --- a/Sources/Workspace/Support/FileChangeStreams.swift +++ b/Sources/Workspace/Support/FileChangeStreams.swift @@ -6,6 +6,14 @@ import Darwin import Glibc #endif +/// Native notifications already carry stored spellings. Parse and rebase their +/// UTF-8 components without a second filesystem lookup (the entry may be gone). +func nativeChangePath(_ name: String, relativeTo root: String) -> WorkspacePath? { + guard let absolute = try? WorkspacePath(name), let root = try? WorkspacePath(root), + absolute.isWithin(root) else { return nil } + return WorkspacePath(normalizing: absolute.components.dropFirst(root.components.count).joined(separator: "/")) +} + enum DirectoryChangeStream { static func observe(_ directory: URL, recursive: Bool = false) throws -> AsyncStream { #if os(macOS) @@ -28,40 +36,63 @@ enum DirectoryChangeStream { source.resume() } #elseif canImport(Glibc) - return AsyncStream { continuation in + return unscopedChanges(try observeSignals(directory, recursive: recursive)) + #else + throw WorkspaceError.unsupported("filesystem change streams are not supported on this platform") + #endif + } + + static func observeSignals(_ directory: URL, recursive: Bool = false) throws -> AsyncStream { + #if os(macOS) + return try FSEventsChangeStream.observeSignals(directory) + #elseif canImport(Glibc) + var failure: Error? + let stream = AsyncStream { continuation in do { - let observation = try LinuxDirectoryObservation( - directory: directory, - recursive: recursive, - continuation: continuation - ) + let observation = try LinuxDirectoryObservation(directory: directory, recursive: recursive, continuation: continuation) continuation.onTermination = { _ in observation.cancel() } } catch { + failure = error continuation.finish() } } + if let failure { throw failure } + return stream #else - throw WorkspaceError.unsupported("filesystem change streams are not supported on this platform") + let stream = try observe(directory, recursive: recursive) + return AsyncStream { continuation in + let task = Task { + for await _ in stream where !Task.isCancelled { continuation.yield(.rescan) } + continuation.finish() + } + continuation.onTermination = { _ in task.cancel() } + } #endif } + } #if canImport(Glibc) private final class LinuxDirectoryObservation: @unchecked Sendable { private let directory: URL private let recursive: Bool - private let continuation: AsyncStream.Continuation + private let continuation: AsyncStream.Continuation private let descriptor: Int32 private let source: DispatchSourceRead private var pathsByWatch: [Int32: Set] = [:] // Keep the root's parent watched so recreating the root can restore coverage even // when it was absent during the IN_DELETE_SELF/IN_MOVE_SELF rescan. private var parentWatch: Int32? + private var pendingIgnored: Set = [] + + private func removeWatch(_ watch: Int32) { + if inotify_rm_watch(descriptor, watch) == 0 { pendingIgnored.insert(watch) } + } init( directory: URL, recursive: Bool, - continuation: AsyncStream.Continuation + continuation: AsyncStream.Continuation ) throws { self.directory = directory self.recursive = recursive @@ -92,6 +123,9 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { var retired = Set() var ignored = Set() var overflow = false + var reconcile = false + var unknown = false + var changed: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] // Drain first, retire each descriptor once, then reconcile against the live tree. // No event performs a scan of the path cache or a scan of descendant prefixes. while true { @@ -106,6 +140,7 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { let recordSize = headerSize + Int(event.len) guard recordSize <= count - offset else { break } defer { offset += recordSize } + if event.mask & UInt32(IN_IGNORED) != 0, pendingIgnored.remove(event.wd) != nil { continue } if event.wd == parentWatch { if event.mask & UInt32(IN_IGNORED) != 0 { parentWatch = nil @@ -116,6 +151,26 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { } } received = true + let nameBytes = bytes[(offset + headerSize)..<(offset + recordSize)].prefix { $0 != 0 } + guard let name = String(bytes: nameBytes, encoding: .utf8) else { + unknown = true + continue + } + if LocalAtomicTemporaryName.matches(name) { continue } + let structural = event.mask & UInt32(IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_DELETE_SELF | IN_MOVE_SELF | IN_IGNORED) != 0 + let isDirectory = name.isEmpty || event.mask & UInt32(IN_ISDIR) != 0 + let scope: FileSystemChangeSignal.Scope = isDirectory ? (structural ? .recursive : .shallow) : .entry + if let watched = pathsByWatch[event.wd] { + for parent in watched { + let absolute = name.isEmpty ? parent.string : parent.string + "/" + name + if let path = nativeChangePath(absolute, relativeTo: directory.path) { + changed[path] = max(changed[path] ?? .entry, scope) + } else { unknown = true } + } + } else { reconcile = true; unknown = true } + if isDirectory && structural { + reconcile = true + } if event.mask & UInt32(IN_IGNORED | IN_DELETE_SELF | IN_MOVE_SELF) != 0 { retired.insert(event.wd) if event.mask & UInt32(IN_IGNORED) != 0 { ignored.insert(event.wd) } @@ -128,24 +183,25 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { if overflow { var watches = Set(pathsByWatch.keys) if let parentWatch { watches.insert(parentWatch) } - for watch in watches where !ignored.contains(watch) { _ = inotify_rm_watch(descriptor, watch) } + for watch in watches where !ignored.contains(watch) { removeWatch(watch) } pathsByWatch.removeAll(keepingCapacity: true) parentWatch = nil } else { for watch in retired { if pathsByWatch.removeValue(forKey: watch) != nil { - if !ignored.contains(watch) { _ = inotify_rm_watch(descriptor, watch) } + if !ignored.contains(watch) { removeWatch(watch) } } } } do { - try addMissingWatches(requireRoot: false) + if reconcile || overflow { try addMissingWatches(requireRoot: false) } } catch { continuation.finish() cancel() return } - continuation.yield() + if overflow || unknown { continuation.yield(.rescan) } + else if !changed.isEmpty { continuation.yield(.paths(changed)) } } private func addMissingWatches(requireRoot: Bool) throws { @@ -190,7 +246,7 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { // A descriptor can serve multiple paths. Remove it only after every live path has // been registered, and only if the new reverse map no longer references that wd. for watch in pathsByWatch.keys where nextByWatch[watch] == nil { - _ = inotify_rm_watch(descriptor, watch) + removeWatch(watch) } pathsByWatch = nextByWatch if parentWatch == nil, directory.path != "/" { @@ -204,3 +260,14 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { } } #endif + +/// Retains compatibility for checkpoint stores and callers of the original Void stream. +func unscopedChanges(_ stream: AsyncStream) -> AsyncStream { + AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in + let task = Task { + for await _ in stream where !Task.isCancelled { continuation.yield() } + continuation.finish() + } + continuation.onTermination = { _ in task.cancel() } + } +} diff --git a/Sources/Workspace/Support/LocalAtomicTemporaryName.swift b/Sources/Workspace/Support/LocalAtomicTemporaryName.swift new file mode 100644 index 0000000..e92140c --- /dev/null +++ b/Sources/Workspace/Support/LocalAtomicTemporaryName.swift @@ -0,0 +1,16 @@ +import Foundation + +/// A process-private namespace prevents filtering user files with the historical +/// `.workspace-` spelling, or similar prefixes. Only our nonce plus a complete +/// canonical UUID is recognized; no filesystem lookup is needed after rename. +enum LocalAtomicTemporaryName { + private static let prefix = ".workspace-atomic-" + UUID().uuidString + "-" + + static func make() -> String { prefix + UUID().uuidString } + + static func matches(_ name: String) -> Bool { + guard name.utf8.starts(with: prefix.utf8) else { return false } + let suffix = String(decoding: name.utf8.dropFirst(prefix.utf8.count), as: UTF8.self) + return UUID(uuidString: suffix)?.uuidString.utf8.elementsEqual(suffix.utf8) == true + } +} diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 45c2bdc..4c81684 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -281,7 +281,7 @@ public actor RuleBasedPermissionAuthorizer: PermissionAuthorizing { /// Backends advertising symlinks must support real-path resolution; unsupported resolution fails closed. /// Creation through an unresolved dangling link also fails closed. Existence checks conservatively /// authorize the target because some backends implement them with dereferencing semantics. -public final class AuthorizedFileSystem: FileSystem, Sendable { +public final class AuthorizedFileSystem: FileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, Sendable { private let base: any FileSystem private let authorizer: any PermissionAuthorizing @@ -333,6 +333,10 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { try await base.createFile(path: path, data: data) } + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { + await base.previewBehavior(at: path) + } + /// See ``FileSystem/capabilities()``. public func capabilities() async -> FileSystemFeatures { await base.capabilities() @@ -406,6 +410,19 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { return try await base.resolveRealPath(path: path) } + package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { + // Potential paths deliberately include dangling targets. Authorize the + // request and backend-resolved result without requiring either to exist. + try await checkDecision(.init(operation: .resolveRealPath, path: path)) + let resolved = try await FileSystemPathResolver(base: base).resolve( + path, features: await base.capabilities(), followFinal: true + ) + if resolved != path { + try await checkDecision(.init(operation: .resolveRealPath, path: resolved)) + } + return resolved + } + /// See ``FileSystem/exists(path:)``. public func exists(path: WorkspacePath) async -> Bool { do { @@ -513,3 +530,37 @@ public final class AuthorizedFileSystem: FileSystem, Sendable { } } } + + +extension AuthorizedFileSystem: FileSystemChangeSource { + public func changes() async throws -> AsyncStream { + unscopedChanges(try await changeSignals()) + } + + public func changeSignals() async throws -> AsyncStream { + guard let source = base as? any FileSystemChangeSource else { + throw WorkspaceError.unsupported("filesystem change notifications are not supported") + } + let signals = try await source.changeSignals() + return AsyncStream { continuation in + let task = Task { + for await signal in signals where !Task.isCancelled { + switch signal { + case .rescan: continuation.yield(.rescan) + case let .paths(paths): + var allowed: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + for (path, scope) in paths { + do { + try await self.authorize(.init(operation: .stat, path: path)) + allowed[path] = scope + } catch { continue } + } + if !allowed.isEmpty { continuation.yield(.paths(allowed)) } + } + } + continuation.finish() + } + continuation.onTermination = { _ in task.cancel() } + } + } +} diff --git a/Sources/Workspace/TransactionTree.swift b/Sources/Workspace/TransactionTree.swift new file mode 100644 index 0000000..1e2a183 --- /dev/null +++ b/Sources/Workspace/TransactionTree.swift @@ -0,0 +1,304 @@ +import Foundation + +/// Immutable nodes and lazy content from a revision, plus changed file bytes. +struct TransactionTree: Sendable { + var nodes: [WorkspacePath: Workspace.MergeNode] + let children: [WorkspacePath: [WorkspacePath]] + var checkpointID: UUID? + var revisionPin: TransactionRevisionPin? + var deltaPaths: Set = [] + var load: @Sendable (WorkspacePath) async throws -> Data + + init(index: RevisionIndex, load: @escaping @Sendable (WorkspacePath) async throws -> Data) { + var nodes: [WorkspacePath: Workspace.MergeNode] = [:] + func visit(_ entry: RevisionIndex.Entry) { + switch entry { + case .missing: break + case let .file(path, size, permissions, id): + nodes[path] = .init(kind: .file, permissions: permissions, contentID: id, size: size) + case let .directory(path, permissions, children): + nodes[path] = .init(kind: .directory, permissions: permissions) + children.forEach(visit) + case let .symbolicLink(path, target, permissions): + nodes[path] = .init(kind: .symlink, permissions: permissions, symlinkTarget: target) + } + } + visit(index.entry) + self.nodes = nodes + var children: [WorkspacePath: [WorkspacePath]] = [:] + for path in nodes.keys where !path.isRoot { children[path.parent, default: []].append(path) } + self.children = children + self.load = load + } + + init(nodes: [WorkspacePath: Workspace.MergeNode], load: @escaping @Sendable (WorkspacePath) async throws -> Data) { + self.nodes = nodes + var children: [WorkspacePath: [WorkspacePath]] = [:] + for path in nodes.keys where !path.isRoot { children[path.parent, default: []].append(path) } + self.children = children + self.load = load + } + + func data(_ path: WorkspacePath) async throws -> Data { + if let data = nodes[path]?.fileData { return data } + return try await load(path) + } + + /// Only affected subtrees need content; directory capture builds its child map once. + func snapshot(at root: WorkspacePath = .root) async throws -> Snapshot { + func materialize(_ path: WorkspacePath) async throws -> Snapshot.Entry { + guard let node = nodes[path] else { return .missing(.init(path: path)) } + switch node.kind { + case .file: return .file(.init(path: path, data: try await data(path), permissions: node.permissions)) + case .symlink: return .symlink(.init(path: path, target: node.symlinkTarget!, permissions: node.permissions)) + case .directory: + var result: [Snapshot.Entry] = [] + for child in (children[path] ?? []).sorted() { result.append(try await materialize(child)) } + return .directory(.init(path: path, permissions: node.permissions, children: result)) + } + } + return Snapshot(rootPath: root, entry: try await materialize(root)) + } + + func changes(to other: Self, paths: Set? = nil, maxTextBytes: Int? = DiffOptions.default.maxTextBytes) async throws -> ChangeSet { + var changes: [FileChange] = [] + var omissions: [TextDiffOmission] = [] + func leaf(_ path: WorkspacePath, in tree: Self) async throws -> Snapshot { + if let node = tree.nodes[path], node.kind == .directory { + return Snapshot(rootPath: path, entry: .directory(.init(path: path, permissions: node.permissions, children: []))) + } + return try await tree.snapshot(at: path) + } + for path in paths ?? Set(nodes.keys).union(other.nodes.keys) where nodes[path] != other.nodes[path] { + let part = try await ChangeSet.compare(before: leaf(path, in: self), after: leaf(path, in: other), maxTextBytes: maxTextBytes) + changes += part.changes + omissions += part.textDiffOmissions + } + return ChangeSet(changes: changes, textDiffOmissions: omissions) + } + + func updating(from fs: any FileSystem, metadata: MetadataIndex, previous: MetadataIndex, policy: Workspace.CheckpointCapturePolicy) async throws -> Self { + var updated = nodes + var changed: Set = [] + for path in nodes.keys where metadata.entries[path] == nil { + updated.removeValue(forKey: path) + changed.insert(path) + } + for (path, entry) in metadata.entries { + let info = entry.info + var node = Workspace.MergeNode(kind: info.kind, permissions: info.permissions, symlinkTarget: entry.target) + if info.kind == .file { + if policy == .trustMetadata, let old = previous.entries[path], entry.canReuseContents(old, capturedAt: previous.capturedAt, exactTimestamps: metadata.exactTimestamps && previous.exactTimestamps), let existing = nodes[path], existing.kind == .file { + node = existing + node.permissions = info.permissions + } else { + let bytes = try await fs.readFile(path: path) + node = .init(kind: .file, permissions: info.permissions, fileData: bytes, size: UInt64(bytes.count)) + } + } + if nodes[path] != node { + updated[path] = node + changed.insert(path) + } + } + var result = Self(nodes: updated, load: load) + result.checkpointID = checkpointID + result.revisionPin = revisionPin + result.deltaPaths = changed + return result + } + + func replacing(with snapshots: [Snapshot]) -> Self { + var updated = nodes + var candidates: Set = [] + for snapshot in snapshots { + let removed = updated[snapshot.rootPath]?.kind == .directory + ? updated.keys.filter { Snapshot.contains(snapshot.rootPath, $0) } : [snapshot.rootPath] + for path in removed { updated.removeValue(forKey: path); candidates.insert(path) } + var replacement: [WorkspacePath: Workspace.MergeNode] = [:] + Workspace.flattenMergeNodes(snapshot.entry, into: &replacement) + candidates.formUnion(replacement.keys) + updated.merge(replacement) { _, new in new } + } + var result = Self(nodes: updated, load: load) + result.checkpointID = checkpointID + result.revisionPin = revisionPin + result.deltaPaths = Set(candidates.filter { nodes[$0] != updated[$0] }) + return result + } + +} + +/// Read-only revision adapter. Directory lookup is indexed once, with no content materialization. +final class TransactionFileSystem: FileSystem, Sendable { + let tree: TransactionTree + let children: [WorkspacePath: [WorkspacePath]] + init(_ tree: TransactionTree) { + self.tree = tree + self.children = tree.children + } + func stat(path: WorkspacePath) async throws -> FileInfo { + guard let node = tree.nodes[path] else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } + return FileInfo(path: path, kind: node.kind, size: node.size, permissions: node.permissions, modificationDate: nil) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + guard try await stat(path: path).kind == .directory else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) } + var result: [DirectoryEntry] = [] + for child in (children[path] ?? []).sorted() { result.append(DirectoryEntry(name: child.name, info: try await stat(path: child))) } + return result + } + func readFile(path: WorkspacePath) async throws -> Data { try await tree.data(path) } + func readSymlink(path: WorkspacePath) async throws -> String { + guard let target = tree.nodes[path]?.symlinkTarget else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EINVAL)) } + return target + } + func exists(path: WorkspacePath) async -> Bool { tree.nodes[path] != nil } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + let pattern = try WorkspacePath(validating: pattern, relativeTo: currentDirectory) + let regex = try NSRegularExpression(pattern: WorkspacePath.globToRegex(pattern.string)) + return tree.nodes.keys.filter { regex.firstMatch(in: $0.string, range: NSRange($0.string.startIndex..., in: $0.string)) != nil }.sorted() + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { throw WorkspaceError.readOnly } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func remove(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { throw WorkspaceError.readOnly } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } +} + +/// A live handle owns the cold base. The parent keeps only a weak reference so +/// abandoning the last handle releases its bytes without an asynchronous cleanup. +final class ColdTransactionBase: Sendable { + let tree: TransactionTree + let metadata: MetadataIndex + init(tree: TransactionTree, metadata: MetadataIndex) { + self.tree = tree + self.metadata = metadata + } +} + +/// Bounded memory plus private immutable overflow blobs for a cold directory base. +/// No public checkpoint, manifest, or eager hash is needed for retained bytes. +final class TransactionBacking: Sendable { + static let memoryBudgetBytes = 256 * 1024 * 1024 + let directory: URL + init() throws { + directory = FileManager.default.temporaryDirectory.appendingPathComponent("Workspace-transaction-\(UUID())") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: false, + attributes: [.posixPermissions: 0o700]) + } + deinit { try? FileManager.default.removeItem(at: directory) } + + static func freeze(from fs: any FileSystem, previous: ColdTransactionBase? = nil, + policy: Workspace.CheckpointCapturePolicy = .trustMetadata) async throws -> ColdTransactionBase { + var metadata = MetadataIndex() + metadata.exactTimestamps = await fs.supportsExactContentTimestamps + var nodes: [WorkspacePath: Workspace.MergeNode] = [:] + var blobs: Set = [] + var spooled: [WorkspacePath: String] = [:] + var backing: TransactionBacking? + var remaining = memoryBudgetBytes + try await fs.walk { info, read, readLink in + let path = info.path + let target = info.kind == .symlink ? try await readLink() : nil + let entry = MetadataIndex.Entry(info: info, target: target) + metadata.entries[path] = entry + var node = Workspace.MergeNode(kind: entry.info.kind, permissions: entry.info.permissions, + symlinkTarget: entry.target) + if entry.info.kind == .file { + if policy == .trustMetadata, let previous, let old = previous.metadata.entries[path], + entry.canReuseContents(old, capturedAt: previous.metadata.capturedAt, + exactTimestamps: metadata.exactTimestamps && previous.metadata.exactTimestamps), + let existing = previous.tree.nodes[path], existing.kind == .file { + node = existing + node.permissions = entry.info.permissions + } else { + let data = try await read(nil) + node = .init(kind: .file, permissions: entry.info.permissions, + fileData: data, size: UInt64(data.count)) + } + if let data = node.fileData, data.count <= remaining { + remaining -= data.count + } else if let data = node.fileData { + if backing == nil { backing = try TransactionBacking() } + let hash = node.contentID! + if blobs.insert(hash).inserted { try data.write(to: backing!.directory.appendingPathComponent(hash)) } + node = .init(kind: .file, permissions: entry.info.permissions, + contentID: hash, size: UInt64(data.count)) + spooled[path] = hash + } + } + nodes[path] = node + } + let overflow = spooled, storage = backing, inheritedLoad = previous?.tree.load + let tree = TransactionTree(nodes: nodes, load: { path in + if let storage, let hash = overflow[path] { + return try Data(contentsOf: storage.directory.appendingPathComponent(hash)) + } + if let inheritedLoad { return try await inheritedLoad(path) } + throw WorkspaceError.readOnly + }) + return ColdTransactionBase(tree: tree, metadata: metadata) + } +} + +/// Revision reads remain lazy and uncached; parent lifetime must not retain read contents. +/// The registry keeps cooperative in-process compaction from deleting lazy bases. +actor TransactionRevisionContent { + let checkpoint: Checkpoint + let store: any CheckpointStore + + init(checkpoint: Checkpoint, store: any CheckpointStore) { + self.checkpoint = checkpoint + self.store = store + } + + func read(_ path: WorkspacePath) async throws -> Data { + guard let data = try await store.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId, + path: path, offset: 0, length: nil) else { + throw WorkspaceError.revisionDataNotFound(checkpoint.snapshotId) + } + return data + } +} + +/// Each reusable base and transaction handle owns an independent pin. Compaction may +/// exclude its own reusable base while retaining handles and other workspace instances. +final class TransactionRevisionPin: Sendable { + let checkpoint: Checkpoint + let token: UUID + init(_ checkpoint: Checkpoint) { + self.checkpoint = checkpoint + token = TransactionRevisionPins.shared.insert(checkpoint) + } + deinit { TransactionRevisionPins.shared.remove(token) } +} + +final class TransactionRevisionPins: @unchecked Sendable { + static let shared = TransactionRevisionPins() + private let lock = NSLock() + private var pins: [UUID: (workspace: UUID, checkpoint: UUID)] = [:] + func insert(_ checkpoint: Checkpoint) -> UUID { + let token = UUID() + lock.withLock { pins[token] = (checkpoint.workspaceId, checkpoint.id) } + return token + } + func remove(_ token: UUID) { _ = lock.withLock { pins.removeValue(forKey: token) } } + func checkpoints(in workspace: UUID, excluding token: UUID? = nil) -> Set { + lock.withLock { Set(pins.filter { $0.key != token && $0.value.workspace == workspace }.map { $0.value.checkpoint }) } + } +} + +/// Serializes revision-pin acquisition with cooperative compaction across workspace instances. +actor TransactionRevisionGate { + static let shared = TransactionRevisionGate() + private var held = false + private var waiters: [CheckedContinuation] = [] + + func withLock(_ body: @Sendable () async throws -> T) async rethrows -> T { + if held { await withCheckedContinuation { waiters.append($0) } } else { held = true } + defer { + if waiters.isEmpty { held = false } else { waiters.removeFirst().resume() } + } + return try await body() + } +} diff --git a/Sources/Workspace/Workspace+Checkpoints.swift b/Sources/Workspace/Workspace+Checkpoints.swift index fbbc794..64a9fe0 100644 --- a/Sources/Workspace/Workspace+Checkpoints.swift +++ b/Sources/Workspace/Workspace+Checkpoints.swift @@ -8,6 +8,7 @@ extension Workspace { try Task.checkCancellation() try await prepareForMutation() try await reconcileCheckpointsWithStore() + let metadata = hasBegunTransaction ? try await MetadataIndex.capture(from: filesystem) : nil let checkpoint = try await store.captureRevision( from: filesystem, draft: CheckpointDraft( @@ -21,6 +22,7 @@ extension Workspace { checkpoints.append(checkpoint) checkpoints.sort(by: checkpointSort) headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) + if let metadata { try? await seedTransactionBase(checkpoint, metadata: metadata) } emitWorkspaceEvent(.checkpoint(checkpoint)) return checkpoint } @@ -52,7 +54,7 @@ extension Workspace { operation: .rollback, changes: recordingChanges(changes, from: previousSnapshot, to: restoredSnapshot) ) - noteWorkspaceSnapshot(restoredSnapshot) + await noteWorkspaceSnapshot(restoredSnapshot) if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } return result } catch { diff --git a/Sources/Workspace/Workspace+Edits.swift b/Sources/Workspace/Workspace+Edits.swift index 2570117..d3aae41 100644 --- a/Sources/Workspace/Workspace+Edits.swift +++ b/Sources/Workspace/Workspace+Edits.swift @@ -10,16 +10,13 @@ extension Workspace { return text } - /// Previews edits against an isolated in-memory copy of the current workspace. + /// Previews edits against an isolated copy-on-write layer over the current workspace. public func preview(_ edits: [Edit]) async throws -> ChangeSet { - let before = try await Snapshot.capture(from: filesystem) - let scratch = InMemoryFileSystem() - try await Snapshot.restore(before, to: scratch) - for edit in edits { - try await Self.apply(edit, to: scratch) - } - let after = try await Snapshot.capture(from: scratch) - return changeSet(from: before, to: after) + await acquireOperation() + defer { releaseOperation() } + let scratch = OverlayFileSystem(previewOf: filesystem) + let workspace = Workspace(fileSystem: scratch, recording: .off) + return try await workspace.apply(edits).changes } /// Applies edits through one canonical execution and recording path. @@ -43,6 +40,7 @@ extension Workspace { do { let paths = try await affectedRoots(for: edit) try await journal.capture(paths, from: filesystem) + if transactionRoots != nil { transactionRoots!.formUnion(paths) } try await Self.apply(edit, to: filesystem) } catch { if policy == .atomic { throw error } @@ -57,7 +55,7 @@ extension Workspace { let recorded = recordingChanges(changes, from: journal.snapshots, to: after) try await appendMutation(operation: .edit, changes: recorded) } - for snapshot in after { noteWorkspaceSnapshot(snapshot) } + for snapshot in after { await noteWorkspaceSnapshot(snapshot) } if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } return EditResult(changes: changes, failures: failures) } catch { diff --git a/Sources/Workspace/Workspace+Internals.swift b/Sources/Workspace/Workspace+Internals.swift index 00ff4cf..fe991f3 100644 --- a/Sources/Workspace/Workspace+Internals.swift +++ b/Sources/Workspace/Workspace+Internals.swift @@ -17,11 +17,12 @@ extension Workspace { } } - func restoreAfterFailure(_ before: [Snapshot], originalError: any Error) async throws -> Never { + func restoreAfterFailure(_ before: [Snapshot], originalError: any Error, restoringSymlinkPermissions: Bool = false) async throws -> Never { var rollbackError: (any Error)? for snapshot in before { do { try await Snapshot.restore(snapshot, to: filesystem) + if restoringSymlinkPermissions { try await restoreTransactionSymlinkPermissions(snapshot.entry) } } catch { if rollbackError == nil { rollbackError = error } } @@ -204,34 +205,81 @@ extension Workspace { guard filesystemObservationTask == nil, let source = filesystem as? any FileSystemChangeSource else { return } - guard let stream = try? await source.changes(), - let initial = try? await Snapshot.capture(from: source) + guard let stream = try? await source.changeSignals(), + let initial = try? await FilesystemObservation.start(on: filesystem, budget: observation.contentCacheBytes) else { return } - observedFilesystemSnapshot = initial + observedFilesystem = initial filesystemObservationTask = Task { [weak self] in - for await _ in stream where !Task.isCancelled { - try? await Task.sleep(for: .milliseconds(40)) - await self?.refreshFilesystemEvents() + for await signal in stream where !Task.isCancelled { + await self?.enqueueFilesystemSignal(signal) } } } - func noteWorkspaceSnapshot(_ snapshot: Snapshot) { + func enqueueFilesystemSignal(_ signal: FileSystemChangeSignal, retry: Bool = false) { guard filesystemObservationTask != nil else { return } - guard var observed = observedFilesystemSnapshot else { return } - observed.entry = Snapshot.replacing(observed.entry, with: snapshot.entry) - observedFilesystemSnapshot = observed + if !retry { + filesystemRefreshFailures = 0 + filesystemRetryMilliseconds = 40 + } + switch (pendingFilesystemSignal, signal) { + case (.rescan, _), (_, .rescan): pendingFilesystemSignal = .rescan + case let (.paths(old), .paths(new)): pendingFilesystemSignal = .paths(old.merging(new, uniquingKeysWith: max)) + case (nil, _): pendingFilesystemSignal = signal + } + guard filesystemRefreshTask == nil, filesystemRefreshFailures < 3 else { return } + let delay = filesystemRetryMilliseconds + filesystemRefreshTask = Task { [weak self] in + do { try await Task.sleep(for: .milliseconds(delay)) } catch { return } + await self?.refreshFilesystemEvents() + } + } + + func noteWorkspaceSnapshot(_ snapshot: Snapshot) async { + guard var observed = observedFilesystem else { return } + observedFilesystem = nil + // No content reads: the edit journal already holds the after image. This + // reconciliation is not atomic with external writers; README documents the + // window between the journal's write and this metadata capture. + guard let metadata = try? await MetadataIndex.capture(from: filesystem, at: snapshot.rootPath) else { + if filesystemObservationTask != nil { observedFilesystem = observed } + return + } + observed.replace(root: snapshot.rootPath, metadata: metadata) + observed.remember(snapshot.entry) + if filesystemObservationTask != nil { observedFilesystem = observed } } func refreshFilesystemEvents() async { await acquireOperation() defer { releaseOperation() } - guard let before = observedFilesystemSnapshot, - let after = try? await Snapshot.capture(from: filesystem) - else { return } - observedFilesystemSnapshot = after - let changes = changeSet(from: before, to: after) - if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } + let signal = pendingFilesystemSignal ?? .rescan + pendingFilesystemSignal = nil + guard var observed = observedFilesystem else { + filesystemRefreshTask = nil + return + } + observedFilesystem = nil + do { + let changes = try await observed.refresh(on: filesystem, signal: signal) + filesystemRetryMilliseconds = 40 + filesystemRefreshFailures = 0 + if filesystemObservationTask != nil { + observedFilesystem = observed + if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } + } + } catch { + if filesystemObservationTask != nil { + observedFilesystem = observed + // Keep the consumed invalidation alongside signals received during the read. + // Bound automatic attempts; a new native signal restarts the retry budget. + filesystemRefreshFailures += 1 + filesystemRetryMilliseconds = min(1_000, filesystemRetryMilliseconds * 2) + enqueueFilesystemSignal(signal, retry: true) + } + } + filesystemRefreshTask = nil + if let pending = pendingFilesystemSignal { enqueueFilesystemSignal(pending, retry: true) } } func checkpointSort(lhs: Checkpoint, rhs: Checkpoint) -> Bool { diff --git a/Sources/Workspace/Workspace+Merge.swift b/Sources/Workspace/Workspace+Merge.swift index d9c8de7..e1ea000 100644 --- a/Sources/Workspace/Workspace+Merge.swift +++ b/Sources/Workspace/Workspace+Merge.swift @@ -2,15 +2,49 @@ import Foundation extension Workspace { /// A flat tree node used by transaction three-way commits. - struct MergeNode: Equatable { + struct MergeNode: Equatable, Sendable { var kind: FileTree.Kind var permissions: POSIXPermissions - var fileData: Data? + let fileData: Data? var symlinkTarget: String? + private var revisionContentID: String? + private let digest: ContentDigest? + var contentID: String? { + get { revisionContentID ?? fileData.flatMap { digest?.value(for: $0) } } + set { revisionContentID = newValue } + } + + /// Shared by value copies of this immutable payload, synchronized for concurrent comparisons. + private final class ContentDigest: @unchecked Sendable { + private let lock = NSLock() + private var cached: String? + func value(for data: Data) -> String { + lock.withLock { + if let cached { return cached } + let value = SHA256.hexDigest(of: data) + cached = value + return value + } + } + } + var size: UInt64 = 0 + + init(kind: FileTree.Kind, permissions: POSIXPermissions, fileData: Data? = nil, + symlinkTarget: String? = nil, contentID: String? = nil, size: UInt64 = 0) { + self.kind = kind + self.permissions = permissions + self.fileData = fileData + self.symlinkTarget = symlinkTarget + self.revisionContentID = contentID + self.digest = fileData == nil || contentID != nil ? nil : ContentDigest() + self.size = size + } static func == (lhs: Self, rhs: Self) -> Bool { - lhs.kind == rhs.kind && lhs.permissions == rhs.permissions && lhs.fileData == rhs.fileData - && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) + guard lhs.kind == rhs.kind, lhs.permissions == rhs.permissions, + lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) else { return false } + if let left = lhs.fileData, let right = rhs.fileData { return left == right } + return lhs.contentID == rhs.contentID } } @@ -19,7 +53,7 @@ extension Workspace { case .missing: return case let .file(file): - nodes[file.path] = MergeNode(kind: .file, permissions: file.permissions, fileData: file.data) + nodes[file.path] = MergeNode(kind: .file, permissions: file.permissions, fileData: file.data, size: UInt64(file.data.count)) case let .symlink(link): nodes[link.path] = MergeNode( kind: .symlink, @@ -39,10 +73,11 @@ extension Workspace { static func structuralMergeConflicts( base: [WorkspacePath: MergeNode], parent: [WorkspacePath: MergeNode], - draft: [WorkspacePath: MergeNode] + draft: [WorkspacePath: MergeNode], + paths candidates: Set? = nil ) -> Set { var conflicts: Set = [] - let paths = Set(base.keys).union(parent.keys).union(draft.keys) + let paths = candidates ?? Set(base.keys).union(parent.keys).union(draft.keys) for path in paths where parent[path] != base[path] || draft[path] != base[path] { var ancestor = path var outermost: WorkspacePath? @@ -62,38 +97,13 @@ extension Workspace { return conflicts } - static func buildEntryTree(from nodes: [WorkspacePath: MergeNode]) throws -> Snapshot.Entry { - guard let root = nodes[.root], root.kind == .directory else { + static func validateMergeNodes(_ nodes: [WorkspacePath: MergeNode]) throws { + guard nodes[.root]?.kind == .directory else { throw WorkspaceError.mutationFailed("merged tree must have a directory root") } - var childNames: [WorkspacePath: [(String, MergeNode)]] = [:] - for (path, node) in nodes where !path.isRoot { - // Checking every immediate parent proves the full ancestry inductively, - // since WorkspacePath parents strictly shorten toward the validated root. - guard nodes[path.parent]?.kind == .directory else { - throw WorkspaceError.mutationFailed("merged node \(path) has no directory parent") - } - childNames[path.parent, default: []].append((path.name, node)) - } - - func subtree(at path: WorkspacePath, node: MergeNode) -> Snapshot.Entry { - switch node.kind { - case .file: - return .file(.init(path: path, data: node.fileData ?? Data(), permissions: node.permissions)) - case .symlink: - return .symlink(.init(path: path, target: node.symlinkTarget ?? "", permissions: node.permissions)) - case .directory: - return .directory( - .init( - path: path, - permissions: node.permissions, - children: (childNames[path] ?? []).sorted { PathName($0.0) < PathName($1.0) }.map { name, child in - subtree(at: path.appending(name), node: child) - } - ) - ) - } + // Immediate parents prove full ancestry because paths shorten toward root. + for path in nodes.keys where !path.isRoot && nodes[path.parent]?.kind != .directory { + throw WorkspaceError.mutationFailed("merged node \(path) has no directory parent") } - return subtree(at: .root, node: root) } } diff --git a/Sources/Workspace/Workspace+TransactionStorage.swift b/Sources/Workspace/Workspace+TransactionStorage.swift new file mode 100644 index 0000000..5a8d074 --- /dev/null +++ b/Sources/Workspace/Workspace+TransactionStorage.swift @@ -0,0 +1,103 @@ +import Foundation + +extension Workspace { + // Snapshot restoration deliberately avoids chmod through symlinks. Backends with + // lchmod semantics can restore the link's own mode safely, including during rollback. + func restoreTransactionSymlinkPermissions(_ entry: Snapshot.Entry) async throws { + switch entry { + case let .symlink(link): + if !(await filesystem.previewBehavior(at: link.path)).followsPermissionSymlinks, + try await filesystem.stat(path: link.path).permissions != link.permissions { + try await filesystem.setPermissions(path: link.path, permissions: link.permissions) + } + case let .directory(directory): + for child in directory.children { try await restoreTransactionSymlinkPermissions(child) } + default: break + } + } + + func enableTransactionJournal() { transactionRoots = [] } + + func transactionTree(over base: TransactionTree) async throws -> TransactionTree { + await acquireOperation() + defer { releaseOperation() } + if let roots = transactionRoots { + var snapshots: [Snapshot] = [] + for path in roots.sorted() where !snapshots.contains(where: { Snapshot.contains($0.rootPath, path) }) { + snapshots.append(try await Snapshot.capture(from: filesystem, at: path)) + } + return base.replacing(with: snapshots) + } + return base.replacing(with: [try await Snapshot.capture(from: filesystem)]) + } + + func storedTransactionTree(_ checkpoint: Checkpoint, in store: any CheckpointStore) async throws -> TransactionTree { + if usesMemoryPersistence, + let snapshot = try await store.loadSnapshot(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) { + var nodes: [WorkspacePath: MergeNode] = [:] + Self.flattenMergeNodes(snapshot.entry, into: &nodes) + var tree = TransactionTree(nodes: nodes, load: { _ in throw WorkspaceError.readOnly }) + tree.checkpointID = checkpoint.id + return tree + } + return try await TransactionRevisionGate.shared.withLock { + let content = TransactionRevisionContent(checkpoint: checkpoint, store: store) + guard let index = try await store.loadRevisionIndex(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) else { + throw WorkspaceError.revisionDataNotFound(checkpoint.snapshotId) + } + var tree = TransactionTree(index: index, load: { try await content.read($0) }) + tree.checkpointID = checkpoint.id + tree.revisionPin = TransactionRevisionPin(checkpoint) + return tree + } + } + + func seedTransactionBase(_ checkpoint: Checkpoint, metadata: MetadataIndex) async throws { + transactionBase = try await storedTransactionTree(checkpoint, in: store) + transactionMetadata = metadata + coldTransactionBase = nil + } + + func persistMutationCheckpoint(tree: TransactionTree, metadata previousMetadata: MetadataIndex, changedRoots: [WorkspacePath], label: String?, origin: Checkpoint.Origin, + operation: Mutation.Operation, changes: ChangeSet) async throws -> Checkpoint { + let mutation = changes.isEmpty ? nil : try await appendMutation(operation: operation, changes: changes) + do { + var metadata = previousMetadata + for root in changedRoots { + let replacement = try await MetadataIndex.capture(from: filesystem, at: root) + for path in metadata.entries.keys where Snapshot.contains(root, path) { metadata.entries.removeValue(forKey: path) } + metadata.entries.merge(replacement.entries) { _, new in new } + } + let checkpoint: Checkpoint + let draft = CheckpointDraft(workspaceID: workspaceId, label: label, preferredParentID: headCheckpointId, origin: origin) + if usesMemoryPersistence { + // The memory store shares unchanged Data with its prior revision. + checkpoint = try await store.saveRevision(tree.snapshot(), draft: draft) + } else { + checkpoint = try await store.captureRevision(from: filesystem, draft: draft) + } + checkpoints.append(checkpoint) + checkpoints.sort(by: checkpointSort) + headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) + // A directory base owns only the newest revision index, never accumulated + // changed-file bytes or a read cache from earlier transactions. Index-loading + // failure must not retract an already durable checkpoint; the next begin freezes anew. + transactionBase = usesMemoryPersistence ? tree : try? await storedTransactionTree(checkpoint, in: store) + transactionBase?.checkpointID = checkpoint.id + transactionMetadata = transactionBase == nil ? nil : metadata + coldTransactionBase = nil + emitWorkspaceEvent(.checkpoint(checkpoint)) + return checkpoint + } catch { + if let mutation { + do { + try await store.removeMutation(sequence: mutation.sequence, workspaceId: workspaceId) + mutations.removeAll { $0.sequence == mutation.sequence } + } catch let cleanupError { + throw WorkspaceRollbackError(originalError: error, rollbackError: cleanupError) + } + } + throw error + } + } +} diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 0e6ffda..5e96d00 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -8,12 +8,13 @@ public actor Workspace { case directory(URL) } - /// Controls file-content verification during checkpoint capture with `.directory` persistence. - /// This policy has no effect on `.memory` persistence. + /// Controls content verification for transactions and directory checkpoint capture. public enum CheckpointCapturePolicy: Sendable, Equatable { - /// Reuse matching size and mtime only when the mtime is safely before the prior capture. + /// Reuse matching size and mtime when mtime is over two seconds older than the prior + /// capture, comparing identity when both entries provide it. Exact-timestamp backends + /// skip the transaction racy window. Missing mtimes always require reading contents. case trustMetadata - /// Read and hash every file, including changes that preserve older timestamps. + /// Read and verify every file, including changes that preserve older timestamps. case verifyContents } @@ -28,6 +29,16 @@ public actor Workspace { public static let `default` = RecordingPolicy.full(maxTextBytes: 1_000_000) } + /// Bounds the old-content cache used for external-change text diffs. + public struct ObservationPolicy: Sendable, Equatable { + public let contentCacheBytes: Int + /// Zero disables content caching. Negative budgets are treated as zero. + public init(contentCacheBytes: Int = 32 * 1024 * 1024) { + self.contentCacheBytes = max(0, contentCacheBytes) + } + public static let `default` = ObservationPolicy() + } + struct EventWatcher { var filter: WorkspaceEvent.Filter var continuation: AsyncStream.Continuation @@ -43,6 +54,7 @@ public actor Workspace { public nonisolated var fileSystem: any FileSystem { filesystem } public nonisolated let recording: RecordingPolicy let store: any CheckpointStore + let checkpointCapture: CheckpointCapturePolicy // Actor isolation alone does not span suspension points. Ownership is handed // directly to the next waiter, preventing newcomers from overtaking the queue. @@ -61,19 +73,34 @@ public actor Workspace { var eventWatchers: [UUID: EventWatcher] = [:] var checkpointObservationTask: Task? var filesystemObservationTask: Task? - var observedFilesystemSnapshot: Snapshot? - - /// Creates a workspace over a filesystem. `checkpointCapture` affects only `.directory` persistence. + var observedFilesystem: FilesystemObservation? + var pendingFilesystemSignal: FileSystemChangeSignal? + var filesystemRefreshTask: Task? + var filesystemRetryMilliseconds = 40 + var filesystemRefreshFailures = 0 + public nonisolated let observation: ObservationPolicy + var hasBegunTransaction = false + var transactionBase: TransactionTree? + var transactionMetadata: MetadataIndex? + weak var coldTransactionBase: ColdTransactionBase? + var transactionRoots: Set? + let usesMemoryPersistence: Bool + + /// Creates a workspace over a filesystem with independent history and capture policies. public init( workspaceID: UUID = UUID(), fileSystem: any FileSystem = InMemoryFileSystem(), persistence: Persistence = .memory, recording: RecordingPolicy = .default, - checkpointCapture: CheckpointCapturePolicy = .trustMetadata + checkpointCapture: CheckpointCapturePolicy = .trustMetadata, + observation: ObservationPolicy = .default ) { + if case .memory = persistence { usesMemoryPersistence = true } else { usesMemoryPersistence = false } self.workspaceId = workspaceID self.filesystem = fileSystem self.recording = recording + self.checkpointCapture = checkpointCapture + self.observation = observation self.store = switch persistence { case .memory: InMemoryCheckpointStore() diff --git a/Sources/Workspace/WorkspaceArchive.swift b/Sources/Workspace/WorkspaceArchive.swift index 17ce946..35fb7e4 100644 --- a/Sources/Workspace/WorkspaceArchive.swift +++ b/Sources/Workspace/WorkspaceArchive.swift @@ -97,7 +97,7 @@ extension Workspace { let recorded = recordingChanges(changes, from: before, to: after) try await appendMutation(operation: .archiveRestore, changes: recorded) } - noteWorkspaceSnapshot(after) + await noteWorkspaceSnapshot(after) if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } return changes } catch { diff --git a/Sources/Workspace/WorkspaceChanges.swift b/Sources/Workspace/WorkspaceChanges.swift index edee407..69d486a 100644 --- a/Sources/Workspace/WorkspaceChanges.swift +++ b/Sources/Workspace/WorkspaceChanges.swift @@ -36,6 +36,10 @@ public struct TextDiffOmission: Sendable, Codable, Equatable { public enum Reason: String, Sendable, Codable { case binary case sizeLimitExceeded + /// The previous contents were not retained in the observation cache. + case observationCacheMiss + /// Reading the current file contents failed; its metadata change is still reported. + case contentReadFailed /// The tool adapter omitted text to fit its encoded output budget. case outputBudgetExceeded } diff --git a/Sources/Workspace/WorkspaceCompaction.swift b/Sources/Workspace/WorkspaceCompaction.swift index 50cbd0c..aebbe03 100644 --- a/Sources/Workspace/WorkspaceCompaction.swift +++ b/Sources/Workspace/WorkspaceCompaction.swift @@ -71,8 +71,25 @@ extension Workspace { defer { releaseOperation() } try Task.checkCancellation() try await ensureLoaded() - let result = try await store.compact(workspaceId: workspaceId, retaining: retention, dryRun: dryRun) + // A reusable base alone does not override retention. Independent transaction + // handles and other workspace instances still protect their lazy revision reads. + let reusablePin = transactionBase?.revisionPin?.token + let store = store + let workspaceId = workspaceId + let result = try await TransactionRevisionGate.shared.withLock { + let effectiveRetention: Retention + if case let .latest(count, labels, preserving) = retention { + let known = Set(try await store.listCheckpoints(workspaceId: workspaceId).map(\.id)) + let pins = TransactionRevisionPins.shared.checkpoints(in: workspaceId, excluding: reusablePin).intersection(known) + effectiveRetention = .latest(count, preservingLabeled: labels, preserving: preserving.union(pins)) + } else { effectiveRetention = retention } + return try await store.compact(workspaceId: workspaceId, retaining: effectiveRetention, dryRun: dryRun) + } if !dryRun { + if let id = transactionBase?.checkpointID, !result.checkpoints.contains(where: { $0.id == id }) { + transactionBase = nil + transactionMetadata = nil + } checkpoints = result.checkpoints headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) } diff --git a/Sources/Workspace/WorkspaceEvent.swift b/Sources/Workspace/WorkspaceEvent.swift index 5a85b96..d5d533e 100644 --- a/Sources/Workspace/WorkspaceEvent.swift +++ b/Sources/Workspace/WorkspaceEvent.swift @@ -48,7 +48,12 @@ extension Workspace { if eventWatchers.isEmpty { filesystemObservationTask?.cancel() filesystemObservationTask = nil - observedFilesystemSnapshot = nil + filesystemRefreshTask?.cancel() + filesystemRefreshTask = nil + filesystemRetryMilliseconds = 40 + filesystemRefreshFailures = 0 + pendingFilesystemSignal = nil + observedFilesystem = nil } } diff --git a/Sources/Workspace/WorkspaceTransaction.swift b/Sources/Workspace/WorkspaceTransaction.swift index 83def33..ee605d2 100644 --- a/Sources/Workspace/WorkspaceTransaction.swift +++ b/Sources/Workspace/WorkspaceTransaction.swift @@ -72,109 +72,113 @@ extension Workspace { public nonisolated let id: UUID private let parent: Workspace - private let draft: Workspace - private let base: Snapshot + private struct Contents: Sendable { + let draft: Workspace + let base: TransactionTree + let baseMetadata: MetadataIndex + let coldBase: ColdTransactionBase? + } + private var contents: Contents? private let baseCheckpointID: UUID? private let label: String? private var state: State = .active private var pendingEdits = 0 private var editsDrained: CheckedContinuation? - init(parent: Workspace, draft: Workspace, base: Snapshot, baseCheckpointID: UUID?, label: String?) { + init(parent: Workspace, draft: Workspace, base: TransactionTree, metadata: MetadataIndex, coldBase: ColdTransactionBase? = nil, baseCheckpointID: UUID?, label: String?) { id = UUID() self.parent = parent - self.draft = draft - self.base = base + self.contents = Contents(draft: draft, base: base, baseMetadata: metadata, coldBase: coldBase) self.baseCheckpointID = baseCheckpointID self.label = label } public func readData(from path: WorkspacePath) async throws -> Data { - try requireReadable() - return try await draft.readData(from: path) + let contents = try requireReadable() + return try await contents.draft.readData(from: path) } public func readText(_ path: WorkspacePath) async throws -> String { - try requireReadable() - return try await draft.readText(path) + let contents = try requireReadable() + return try await contents.draft.readText(path) } public func readJSON( _ type: T.Type = T.self, from path: WorkspacePath ) async throws -> T { - try requireReadable() - return try await draft.readJSON(type, from: path) + let contents = try requireReadable() + return try await contents.draft.readJSON(type, from: path) } public func exists(_ path: WorkspacePath) async throws -> Bool { - try requireReadable() - return await draft.exists(path) + let contents = try requireReadable() + return await contents.draft.exists(path) } public func info(_ path: WorkspacePath) async throws -> FileInfo { - try requireReadable() - return try await draft.info(path) + let contents = try requireReadable() + return try await contents.draft.info(path) } public func list(_ path: WorkspacePath) async throws -> [DirectoryEntry] { - try requireReadable() - return try await draft.list(path) + let contents = try requireReadable() + return try await contents.draft.list(path) } public func glob(_ pattern: String, currentDirectory: WorkspacePath = .root) async throws -> [WorkspacePath] { - try requireReadable() - return try await draft.glob(pattern, currentDirectory: currentDirectory) + let contents = try requireReadable() + return try await contents.draft.glob(pattern, currentDirectory: currentDirectory) } public func tree(at path: WorkspacePath = .root, maxDepth: Int? = nil) async throws -> FileTree { - try requireReadable() - return try await draft.tree(at: path, maxDepth: maxDepth) + let contents = try requireReadable() + return try await contents.draft.tree(at: path, maxDepth: maxDepth) } public func search(_ request: SearchRequest) async throws -> SearchResult { - try requireReadable() - return try await draft.search(request) + let contents = try requireReadable() + return try await contents.draft.search(request) } public func preview(_ edits: [Edit]) async throws -> ChangeSet { - try requireReadable() - return try await draft.preview(edits) + let contents = try requireReadable() + return try await contents.draft.preview(edits) } @discardableResult public func apply(_ edits: [Edit], policy: EditPolicy = .atomic) async throws -> EditResult { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.apply(edits, policy: policy) + return try await contents.draft.apply(edits, policy: policy) } @discardableResult public func writeText(_ path: WorkspacePath, _ content: String) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.writeText(path, content) + return try await contents.draft.writeText(path, content) } @discardableResult public func appendText(_ path: WorkspacePath, _ content: String) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.appendText(path, content) + return try await contents.draft.appendText(path, content) } @discardableResult public func writeData(_ data: Data, to path: WorkspacePath) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.writeData(data, to: path) + return try await contents.draft.writeData(data, to: path) } @discardableResult public func appendData(_ path: WorkspacePath, _ data: Data) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.appendData(path, data) + return try await contents.draft.appendData(path, data) } @discardableResult @@ -183,73 +187,73 @@ extension Workspace { to path: WorkspacePath, prettyPrinted: Bool = true ) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.writeJSON(value, to: path, prettyPrinted: prettyPrinted) + return try await contents.draft.writeJSON(value, to: path, prettyPrinted: prettyPrinted) } @discardableResult public func createDirectory(_ path: WorkspacePath, recursive: Bool = true) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.createDirectory(path, recursive: recursive) + return try await contents.draft.createDirectory(path, recursive: recursive) } @discardableResult public func remove(_ path: WorkspacePath, recursive: Bool = true) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.remove(path, recursive: recursive) + return try await contents.draft.remove(path, recursive: recursive) } @discardableResult public func copy(from source: WorkspacePath, to destination: WorkspacePath, recursive: Bool = true) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.copy(from: source, to: destination, recursive: recursive) + return try await contents.draft.copy(from: source, to: destination, recursive: recursive) } @discardableResult public func move(from source: WorkspacePath, to destination: WorkspacePath) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.move(from: source, to: destination) + return try await contents.draft.move(from: source, to: destination) } @discardableResult public func createSymbolicLink(_ path: WorkspacePath, target: String) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.createSymbolicLink(path, target: target) + return try await contents.draft.createSymbolicLink(path, target: target) } @discardableResult public func createHardLink(_ path: WorkspacePath, target: WorkspacePath) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.createHardLink(path, target: target) + return try await contents.draft.createHardLink(path, target: target) } @discardableResult public func setPermissions(_ permissions: POSIXPermissions, at path: WorkspacePath) async throws -> ChangeSet { - try beginEdit() + let contents = try beginEdit() defer { endEdit() } - return try await draft.setPermissions(permissions, at: path) + return try await contents.draft.setPermissions(permissions, at: path) } /// Returns all base-to-draft changes without mutating the parent workspace. public func preview() async throws -> ChangeSet { - try requireReadable() - let current = try await draft.revisionSnapshot(.current) - return ChangeSet.compare(before: base, after: current, maxTextBytes: DiffOptions.default.maxTextBytes) + let contents = try requireReadable() + let current = try await contents.draft.transactionTree(over: contents.base) + return try await contents.base.changes(to: current, paths: current.deltaPaths) } /// Attempts to commit this transaction. Conflicts and safely rolled-back failures /// leave it active for retry. A rollback failure permanently closes it. public func commit(strategy: CommitStrategy = .threeWay) async throws -> Commit { - try requireActive() + let contents = try requireActive() state = .committing do { // An edit admitted before commit may still be suspended in the draft. @@ -258,34 +262,39 @@ extension Workspace { await withCheckedContinuation { editsDrained = $0 } } try Task.checkCancellation() - let draftSnapshot = try await draft.revisionSnapshot(.current) + let draftSnapshot = try await contents.draft.transactionTree(over: contents.base) let result = try await parent.commitTransaction( - base: base, + base: contents.base, draft: draftSnapshot, + baseMetadata: contents.baseMetadata, baseCheckpointID: baseCheckpointID, label: label, strategy: strategy ) state = result.conflicts.isEmpty ? .committed : .active + if state == .committed { self.contents = nil } return result } catch { // The parent's gate covers restoration too. An ordinary error means // no mutation occurred or compensation succeeded. Failed compensation // may have left filesystem or history changes behind: never allow retry. state = error is WorkspaceRollbackError ? .failed : .active + if state == .failed { self.contents = nil } throw error } } /// Permanently discards the transaction. public func discard() throws { - try requireActive() + _ = try requireActive() state = .discarded + contents = nil } - private func beginEdit() throws { - try requireActive() + private func beginEdit() throws -> Contents { + let contents = try requireActive() pendingEdits += 1 + return contents } private func endEdit() { @@ -296,13 +305,15 @@ extension Workspace { } } - private func requireReadable() throws { - guard state == .active || state == .committing else { throw TransactionError.inactive } + private func requireReadable() throws -> Contents { + guard state == .active || state == .committing, let contents else { throw TransactionError.inactive } + return contents } - private func requireActive() throws { + private func requireActive() throws -> Contents { if state == .committing { throw TransactionError.committing } - guard state == .active else { throw TransactionError.inactive } + guard state == .active, let contents else { throw TransactionError.inactive } + return contents } } @@ -318,18 +329,48 @@ extension Workspace { try Task.checkCancellation() try await ensureLoaded() try await reconcileCheckpointsWithStore() - let base = try await Snapshot.capture(from: filesystem) - let draftFileSystem = InMemoryFileSystem() - try await Snapshot.restore(base, to: draftFileSystem) + hasBegunTransaction = true + let metadata: MetadataIndex + var base: TransactionTree + let coldBase: ColdTransactionBase? + if !usesMemoryPersistence && transactionBase == nil { + let frozen = try await TransactionBacking.freeze(from: filesystem, + previous: coldTransactionBase, policy: checkpointCapture) + base = frozen.tree + metadata = frozen.metadata + coldBase = frozen + coldTransactionBase = frozen + } else { + metadata = try await MetadataIndex.capture(from: filesystem) + coldBase = nil + if let previous = transactionBase, let previousMetadata = transactionMetadata { + base = try await previous.updating(from: filesystem, metadata: metadata, previous: previousMetadata, policy: checkpointCapture) + } else { + var nodes: [WorkspacePath: MergeNode] = [:] + for (path, entry) in metadata.entries { + let bytes = entry.info.kind == .file ? try await filesystem.readFile(path: path) : nil + nodes[path] = MergeNode(kind: entry.info.kind, permissions: entry.info.permissions, + fileData: bytes, symlinkTarget: entry.target, size: UInt64(bytes?.count ?? 0)) + } + base = TransactionTree(nodes: nodes, load: { _ in throw WorkspaceError.readOnly }) + } + transactionBase = base + transactionMetadata = metadata + } + if let pin = base.revisionPin { base.revisionPin = TransactionRevisionPin(pin.checkpoint) } + let draftFileSystem = OverlayFileSystem(over: TransactionFileSystem(base), semanticsOf: filesystem) let draft = Workspace( fileSystem: draftFileSystem, persistence: .memory, recording: recording ) + await draft.enableTransactionJournal() return Transaction( parent: self, draft: draft, base: base, + metadata: metadata, + coldBase: coldBase, baseCheckpointID: headCheckpointId, label: label ) @@ -356,8 +397,9 @@ extension Workspace { } func commitTransaction( - base: Snapshot, - draft: Snapshot, + base: TransactionTree, + draft: TransactionTree, + baseMetadata: MetadataIndex, baseCheckpointID: UUID?, label: String?, strategy: Transaction.CommitStrategy @@ -367,18 +409,15 @@ extension Workspace { try Task.checkCancellation() try await prepareForMutation() try await reconcileCheckpointsWithStore() - let current = try await Snapshot.capture(from: filesystem) - let preview = ChangeSet.compare(before: base, after: draft, maxTextBytes: DiffOptions.default.maxTextBytes) - - var baseNodes: [WorkspacePath: MergeNode] = [:] - var currentNodes: [WorkspacePath: MergeNode] = [:] - var draftNodes: [WorkspacePath: MergeNode] = [:] - Self.flattenMergeNodes(base.entry, into: &baseNodes) - Self.flattenMergeNodes(current.entry, into: ¤tNodes) - Self.flattenMergeNodes(draft.entry, into: &draftNodes) - - if strategy == .strict, currentNodes != baseNodes { - let changed = Set(baseNodes.keys).union(currentNodes.keys).filter { baseNodes[$0] != currentNodes[$0] } + let metadata = try await MetadataIndex.capture(from: filesystem) + let current = try await base.updating(from: filesystem, metadata: metadata, previous: baseMetadata, policy: checkpointCapture) + let preview = try await base.changes(to: draft, paths: draft.deltaPaths) + let baseNodes = base.nodes + let currentNodes = current.nodes + let draftNodes = draft.nodes + + if strategy == .strict, !current.deltaPaths.isEmpty { + let changed = current.deltaPaths return Transaction.Commit( preview: preview, appliedChanges: ChangeSet(), @@ -388,9 +427,10 @@ extension Workspace { } var merged = currentNodes - let structuralPaths = Self.structuralMergeConflicts(base: baseNodes, parent: currentNodes, draft: draftNodes) + let changedPaths = current.deltaPaths.union(draft.deltaPaths) + let structuralPaths = Self.structuralMergeConflicts(base: baseNodes, parent: currentNodes, draft: draftNodes, paths: changedPaths) var conflicts = structuralPaths.sorted().map { Transaction.Conflict(path: $0, kind: .structural) } - for path in Set(baseNodes.keys).union(currentNodes.keys).union(draftNodes.keys).sorted() { + for path in changedPaths.sorted() { // One structural conflict describes the entire affected subtree; // suppress redundant node conflicts beneath it. var ancestor = path @@ -416,8 +456,8 @@ extension Workspace { .init( path: path, kind: kind, - parentDiff: Self.conflictDiffForTransaction(from: baseNode, to: parentNode), - transactionDiff: Self.conflictDiffForTransaction(from: baseNode, to: transactionNode) + parentDiff: try await Self.conflictDiffForTransaction(path: path, from: base, to: current), + transactionDiff: try await Self.conflictDiffForTransaction(path: path, from: base, to: draft) ) ) } @@ -430,9 +470,11 @@ extension Workspace { ) } - // Reconstruction validates the whole flat map before the first parent write. - let mergedSnapshot = Snapshot(rootPath: .root, entry: try Self.buildEntryTree(from: merged)) - let applied = ChangeSet.compare(before: current, after: mergedSnapshot, maxTextBytes: DiffOptions.default.maxTextBytes) + // Validate ancestry before the first parent write. + let mergedNodes = merged + let mergedTree = TransactionTree(nodes: mergedNodes, load: current.load) + try Self.validateMergeNodes(merged) + let applied = try await current.changes(to: mergedTree, paths: draft.deltaPaths) guard !applied.isEmpty else { return Transaction.Commit(preview: preview, appliedChanges: applied, checkpoint: nil, conflicts: []) } @@ -456,36 +498,38 @@ extension Workspace { var after: [Snapshot] = [] for path in restoreRoots.sorted() { if before.contains(where: { Snapshot.contains($0.rootPath, path) }) { continue } - let oldEntry = Self.snapshotEntry(at: path, in: current.entry) ?? .missing(.init(path: path)) - let newEntry = Self.snapshotEntry(at: path, in: mergedSnapshot.entry) ?? .missing(.init(path: path)) - before.append(Snapshot(rootPath: path, entry: oldEntry)) - after.append(Snapshot(rootPath: path, entry: newEntry)) + before.append(try await current.snapshot(at: path)) + after.append(try await mergedTree.snapshot(at: path)) } do { - for snapshot in after { try await Snapshot.restore(snapshot, to: filesystem) } + for snapshot in after { + try await Snapshot.restore(snapshot, to: filesystem) + try await restoreTransactionSymlinkPermissions(snapshot.entry) + } let checkpoint = try await persistMutationCheckpoint( - snapshot: mergedSnapshot, + tree: mergedTree, + metadata: metadata, + changedRoots: after.map(\.rootPath), label: label, origin: .transaction(base: baseCheckpointID), operation: .transaction, - changes: recordingChanges(applied, from: current, to: mergedSnapshot) + changes: recordingChanges(applied, from: before, to: after) ) - for snapshot in after { noteWorkspaceSnapshot(snapshot) } + for snapshot in after { await noteWorkspaceSnapshot(snapshot) } emitWorkspaceEvent(.changes(applied)) return Transaction.Commit(preview: preview, appliedChanges: applied, checkpoint: checkpoint, conflicts: []) } catch { - try await restoreAfterFailure(before, originalError: error) + try await restoreAfterFailure(before, originalError: error, restoringSymlinkPermissions: true) } } - private static func conflictDiffForTransaction(from base: MergeNode?, to side: MergeNode?) -> TextDiff? { - let baseData = base?.fileData ?? Data() - let sideData = side?.fileData ?? Data() + private static func conflictDiffForTransaction(path: WorkspacePath, from base: TransactionTree, to side: TransactionTree) async throws -> TextDiff? { + let baseData = base.nodes[path]?.kind == .file ? try await base.data(path) : Data() + let sideData = side.nodes[path]?.kind == .file ? try await side.data(path) : Data() guard let baseText = String(data: baseData, encoding: .utf8), let sideText = String(data: sideData, encoding: .utf8) else { return nil } let diff = TextDiff.lineBased(from: baseText, to: sideText) return diff.hunks.isEmpty ? nil : diff } - } diff --git a/Tests/WorkspaceTests/ChangeScalingTests.swift b/Tests/WorkspaceTests/ChangeScalingTests.swift new file mode 100644 index 0000000..3aa9764 --- /dev/null +++ b/Tests/WorkspaceTests/ChangeScalingTests.swift @@ -0,0 +1,264 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Change-scaled operations") +struct ChangeScalingTests { + @Test(arguments: [false, true]) + func `local transactions and observation scan metadata through one bulk walk`(edit: Bool) async throws { + let root = try TestSupport.temporaryDirectory("MetadataWalk") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + for i in 0..<40 { + try await local.writeFile(path: WorkspacePath("/tree/d\(i)/file"), data: Data("old".utf8), append: false) + } + let counted = HookableFileSystem(base: local, forwardsPermissions: true, scopedWalk: true) + let storage = try TestSupport.temporaryDirectory("ColdMetadataWalk") + defer { TestSupport.remove(storage) } + let directoryWorkspace = Workspace(fileSystem: counted, persistence: .directory(storage)) + let cold = try await directoryWorkspace.beginTransaction() + #expect(await counted.walks == [.root]) + #expect(await counted.reads.isEmpty) // Contents come from scoped walk callbacks. + #expect(await counted.statCount == 0) + #expect(await counted.listings.isEmpty) + #expect(try await cold.readText("/tree/d0/file") == "old") + try await cold.discard() + await counted.resetMetadataCounts() + let workspace = Workspace(fileSystem: counted) + let transaction = try await workspace.beginTransaction() + #expect(await counted.walks == [.root]) + #expect(await counted.statCount == 0) + #expect(await counted.listings.isEmpty) + + if edit { try await transaction.writeText("/tree/d0/file", "new") } + await counted.resetMetadataCounts() + #expect(try await transaction.commit().applied) + // Changed leaves use stat for their scoped reconciliation after restoration. + #expect(await counted.walks == [.root]) + #expect(await counted.statCount < 10) + #expect(await counted.listings.isEmpty) + + var observation = try await FilesystemObservation.start(on: counted, budget: 1024) + try await local.writeFile(path: "/tree/d1/file", data: Data("external".utf8), append: false) + await counted.resetMetadataCounts() + #expect(try await observation.refresh(on: counted, signal: .rescan).touchedPaths == ["/tree/d1/file"]) + #expect(await counted.walks == [.root]) + #expect(await counted.statCount == 0) + #expect(await counted.listings.isEmpty) + + try await local.writeFile(path: "/tree/new/deep/file", data: Data("added".utf8), append: false) + await counted.resetMetadataCounts() + let changes = try await observation.refresh(on: counted, signal: .paths(["/tree": .shallow])) + #expect(changes.touchedPaths == ["/tree/new", "/tree/new/deep", "/tree/new/deep/file"]) + #expect(await counted.walks == ["/tree/new"]) + #expect(await counted.listings == ["/tree"]) + await counted.resetMetadataCounts() + #expect(try await observation.refresh(on: counted, signal: .paths(["/tree": .recursive])).isEmpty) + #expect(await counted.walks == ["/tree"]) + #expect(await counted.listings.isEmpty) + } + + @Test + func `preview excludes untouched symlinks and reads only affected contents`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/edit", data: Data("old".utf8), append: false) + try await local.writeFile(path: "/unrelated", data: Data(repeating: 1, count: 2_000_000), append: false) + try await local.createSymlink(path: "/link", target: "unrelated") + try await local.createSymlink(path: "/dangling", target: "absent") + let counted = HookableFileSystem(base: local) + let workspace = Workspace(fileSystem: counted) + let preview = try await workspace.preview([.writeText("/edit", "new")]) + #expect(preview.touchedPaths == ["/edit"]) + #expect(await counted.reads.allSatisfy { $0 == "/edit" }) + #expect(try await local.readSymlink(path: "/link") == "unrelated") + #expect(try await local.readSymlink(path: "/dangling") == "absent") + #expect(try await workspace.readText("/edit") == "old") + #expect(try await workspace.writeText("/edit", "new") == preview) + } + + @Test(arguments: [false, true]) + func `checkpoint based transactions keep unread files frozen without rereading parent contents`(directory: Bool) async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let fs = HookableFileSystem(base: try LocalFileSystem(root: root), forwardsPermissions: true) + try await fs.base.writeFile(path: "/edit", data: Data("old".utf8), append: false) + try await fs.base.writeFile(path: "/unread", data: Data("frozen".utf8), append: false) + try await fs.base.writeFile(path: "/untouched", data: Data(repeating: 1, count: 1_000_000), append: false) + let workspace = Workspace(fileSystem: fs, persistence: directory ? .directory(storage) : .memory) + for name in ["edit", "unread", "untouched"] { + try FileManager.default.setAttributes([.modificationDate: Date(timeIntervalSince1970: 1_000)], + ofItemAtPath: root.appendingPathComponent(name).path) + } + // First use freezes the base; subsequent checkpoints may prime transaction reuse. + try await workspace.beginTransaction().discard() + _ = try await workspace.createCheckpoint() + let readsBefore = await fs.reads.count + let transaction = try await workspace.beginTransaction() + #expect(await fs.reads.count == readsBefore) + // The draft has never read this file. Mutate outside the Workspace gate. + try await fs.base.writeFile(path: "/unread", data: Data("external".utf8), append: false) + #expect(try await transaction.readText("/unread") == "frozen") + try await transaction.writeText("/edit", "draft") + #expect(try await transaction.preview().touchedPaths == ["/edit"]) + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText("/unread") == "external") + #expect(try await workspace.readText("/edit") == "draft") + // Directory-store capture has its own conservative timestamp verification; + // the memory case isolates transaction reads from that independent policy. + if !directory { + #expect(await fs.reads.dropFirst(readsBefore).allSatisfy { $0 != "/untouched" }) + } + } + + @Test + func `compaction preserves a lazy transaction base until the handle releases it`() async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let workspace = Workspace(persistence: .directory(storage)) + try await workspace.writeText("/unread", "frozen") + try await workspace.beginTransaction().discard() + let checkpoint = try await workspace.createCheckpoint() + let transaction = try await workspace.beginTransaction() + try await workspace.writeText("/unread", "parent") + _ = try await workspace.createCheckpoint() + let report = try await workspace.compact(retaining: .latest(1, preservingLabeled: false)) + #expect(!report.removedCheckpointIDs.contains(checkpoint.id)) + #expect(try await transaction.readText("/unread") == "frozen") + try await transaction.writeText("/new", "draft") + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText("/unread") == "parent") + } + + @Test(arguments: [0, 8], [false, true]) + func `external changes respect the content budget and never enter history`(budget: Int, unknownPaths: Bool) async throws { + let fs = SignallingFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("old\n".utf8), append: false) + try await fs.base.writeFile(path: "/large", data: Data(repeating: 120, count: 100_000), append: false) + let workspace = Workspace(fileSystem: fs, observation: .init(contentCacheBytes: budget)) + let stream = await workspace.events(.init(includeCheckpoints: false)) + let initialReads = await fs.base.reads.count + let initialStats = await fs.base.statCount + #expect(await fs.base.reads.allSatisfy { $0 == "/a" }) + try await fs.base.writeFile(path: "/a", data: Data("new\n".utf8), append: false) + for _ in 0..<20 { fs.signal(unknownPaths ? .rescan : .paths(["/a": .entry])) } + guard case let .changes(changes)? = await event(in: stream) else { + Issue.record("missing external event") + return + } + #expect(changes.touchedPaths == ["/a"]) + #expect((changes.changes[0].diff != nil) == (budget > 0)) + #expect(changes.textDiffOmissions == (budget == 0 ? [.init(path: "/a", reason: .observationCacheMiss)] : [])) + #expect(await fs.base.reads.dropFirst(initialReads).allSatisfy { $0 == "/a" }) + #expect(await fs.base.reads.count - initialReads <= 1) + #expect(try await workspace.history().isEmpty) + #expect(await event(in: stream, timeout: .milliseconds(150)) == nil) + #expect(await fs.base.statCount - initialStats <= 2) + } + + @Test + func `observation distinguishes replacements removals and metadata only changes`() async throws { + let fs = InMemoryFileSystem() + try await fs.createDirectory(path: "/d", recursive: true) + try await fs.writeFile(path: "/d/file", data: Data("old\n".utf8), append: false) + try await fs.createSymlink(path: "/link", target: "d/file") + var observation = try await FilesystemObservation.start(on: fs, budget: 100) + let before = try await Snapshot.capture(from: fs) + try await fs.remove(path: "/d", recursive: true) + try await fs.writeFile(path: "/d", data: Data("replacement\n".utf8), append: false) + try await fs.remove(path: "/link", recursive: false) + try await fs.createSymlink(path: "/link", target: "d") + try await fs.setPermissions(path: .root, permissions: POSIXPermissions(0o700)) + let expected = ChangeSet.compare(before: before, after: try await Snapshot.capture(from: fs), maxTextBytes: DiffOptions.default.maxTextBytes) + #expect(try await observation.refresh(on: fs, signal: .rescan) == expected) + #expect(try await observation.refresh(on: fs, signal: .rescan).isEmpty) + } + + @Test + func `content read failure omits only that diff and follows hard link aliases`() async throws { + let fs = HookableFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("old".utf8), append: false) + try await fs.base.createHardLink(path: "/alias", target: "/a") + try await fs.base.writeFile(path: "/z", data: Data("old".utf8), append: false) + var observation = try await FilesystemObservation.start(on: fs, budget: 100) + let before = try await Snapshot.capture(from: fs.base) + try await fs.base.writeFile(path: "/a", data: Data("-new".utf8), append: true) + try await fs.base.writeFile(path: "/z", data: Data("changed".utf8), append: false) + await fs.onRead { _ in await fs.onRead { _ in throw TestFileSystemError.injected } } + let changes = try await observation.refresh(on: fs, signal: .paths(["/a": .entry, "/z": .entry])) + let expected = ChangeSet.compare(before: before, after: try await Snapshot.capture(from: fs.base), maxTextBytes: DiffOptions.default.maxTextBytes) + #expect(changes.touchedPaths == expected.touchedPaths) + #expect(changes.textDiffOmissions == [.init(path: "/alias", reason: .contentReadFailed)]) + #expect(changes.changes.filter { $0.path != "/alias" } == expected.changes.filter { $0.path != "/alias" }) + } + + @Test(arguments: [0, -1]) + func `disabled observation caching does not read even empty files`(budget: Int) async throws { + let fs = SignallingFileSystem() + try await fs.base.writeFile(path: "/empty", data: Data(), append: false) + let workspace = Workspace(fileSystem: fs, observation: .init(contentCacheBytes: budget)) + let stream = await workspace.events(.init(includeCheckpoints: false)) + #expect(await fs.base.reads.isEmpty) + withExtendedLifetime(stream) {} + } + + @Test + func `observation evicts the least recently used contents`() async throws { + let fs = InMemoryFileSystem() + try await fs.writeFile(path: "/a", data: Data("old\n".utf8), append: false) + try await fs.writeFile(path: "/b", data: Data("old\n".utf8), append: false) + var observation = try await FilesystemObservation.start(on: fs, budget: 8) + // An unchanged path signal still uses the cached contents for byte comparison. + #expect(try await observation.refresh(on: fs, signal: .paths(["/a": .entry])).isEmpty) + try await fs.writeFile(path: "/c", data: Data("new\n".utf8), append: false) + _ = try await observation.refresh(on: fs, signal: .paths(["/c": .entry])) + try await fs.writeFile(path: "/a", data: Data("new\n".utf8), append: false) + let recent = try await observation.refresh(on: fs, signal: .paths(["/a": .entry])) + #expect(recent.changes.first?.diff != nil) + try await fs.writeFile(path: "/b", data: Data("new\n".utf8), append: false) + let evicted = try await observation.refresh(on: fs, signal: .paths(["/b": .entry])) + #expect(evicted.textDiffOmissions == [.init(path: "/b", reason: .observationCacheMiss)]) + } + + private func event(in stream: AsyncStream, timeout: Duration = .seconds(2)) async -> WorkspaceEvent? { + await withTaskGroup(of: WorkspaceEvent?.self) { group in + group.addTask { var iterator = stream.makeAsyncIterator(); return await iterator.next() } + group.addTask { try? await Task.sleep(for: timeout); return nil } + let value = await group.next() ?? nil + group.cancelAll() + return value + } + } +} + +final class SignallingFileSystem: FileSystemChangeSource, ExactTimestampFileSystem, Sendable { + var hasExactContentTimestamps: Bool { get async { await base.supportsExactContentTimestamps } } + let base: HookableFileSystem + let source: (any FileSystemChangeSource)? + let stream: AsyncStream + let continuation: AsyncStream.Continuation + init(base: any FileSystem = InMemoryFileSystem(), source: (any FileSystemChangeSource)? = nil) { + self.base = HookableFileSystem(base: base) + self.source = source + (stream, continuation) = AsyncStream.makeStream() + } + func signal(_ signal: FileSystemChangeSignal) { continuation.yield(signal) } + func changeSignals() async throws -> AsyncStream { + if let source { return try await source.changeSignals() } + return stream + } + func changes() async throws -> AsyncStream { unscopedChanges(stream) } + func stat(path: WorkspacePath) async throws -> FileInfo { try await base.stat(path: path) } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func readFile(path: WorkspacePath) async throws -> Data { try await base.readFile(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { try await base.glob(pattern: pattern, currentDirectory: currentDirectory) } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await base.writeFile(path: path, data: data, append: append) } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await base.remove(path: path, recursive: recursive) } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) } +} diff --git a/Tests/WorkspaceTests/ChangeSoundnessTests.swift b/Tests/WorkspaceTests/ChangeSoundnessTests.swift new file mode 100644 index 0000000..e53e579 --- /dev/null +++ b/Tests/WorkspaceTests/ChangeSoundnessTests.swift @@ -0,0 +1,334 @@ +import Foundation +import Testing +@testable import Workspace + +/// Models coarse or preserved metadata independently of the underlying in-memory clock. +final class CoarseFileSystem: FileSystem, FileSystemChangeSource, Sendable { + let base = InMemoryFileSystem() + let stream: AsyncStream + let continuation: AsyncStream.Continuation + enum Mode: Sendable, CaseIterable { case coarse, racy, verified } + let mode: Mode + let timestamp: Date + init(mode: Mode = .coarse, timestamp: Date? = nil) { + self.mode = mode + self.timestamp = timestamp ?? (mode == .verified ? Date(timeIntervalSince1970: 1_000) : Date(timeIntervalSince1970: floor(Date().timeIntervalSince1970))) + (stream, continuation) = AsyncStream.makeStream() + } + func signal(_ s: FileSystemChangeSignal) { continuation.yield(s) } + func changeSignals() async throws -> AsyncStream { stream } + func changes() async throws -> AsyncStream { unscopedChanges(stream) } + func coarse(_ info: FileInfo) -> FileInfo { + var i = info + i.identity = mode == .coarse ? nil : FileIdentity(device: 1, inode: info.path == "/a" ? 1 : 2) + i.modificationDate = timestamp + return i + } + func stat(path: WorkspacePath) async throws -> FileInfo { coarse(try await base.stat(path: path)) } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + try await base.listDirectory(path: path).map { DirectoryEntry(name: $0.name, info: coarse($0.info)) } + } + func readFile(path: WorkspacePath) async throws -> Data { try await base.readFile(path: path) } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { try await base.glob(pattern: pattern, currentDirectory: currentDirectory) } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await base.writeFile(path: path, data: data, append: append) } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await base.remove(path: path, recursive: recursive) } + func move(from s: WorkspacePath, to d: WorkspacePath) async throws { try await base.move(from: s, to: d) } + func copy(from s: WorkspacePath, to d: WorkspacePath, recursive: Bool) async throws { try await base.copy(from: s, to: d, recursive: recursive) } + func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { try await base.setPermissions(path: path, permissions: permissions) } +} + +@Suite("Transaction and observation soundness") +struct ChangeSoundnessTests { + @Test(arguments: ["discard", "deallocate", "commit"]) + func `cold transaction bytes live only as long as active handles`(finish: String) async throws { + let storage = try TestSupport.temporaryDirectory("ColdBaseLifetime") + defer { TestSupport.remove(storage) } + let allocation = TrackedColdAllocation() + let base = InMemoryFileSystem() + try await base.writeFile(path: "/a", data: Data(repeating: 42, count: 4096), append: false) + let fs = HookableFileSystem(base: base, forwardsPermissions: true, transformRead: { allocation.wrapFirst($0) }) + let ws = Workspace(fileSystem: fs, persistence: .directory(storage)) + var transaction: Workspace.Transaction? = try await ws.beginTransaction() + #expect(allocation.live == 1) + // A second live handle shares the frozen bytes, even if the first closes. + let sibling = try await ws.beginTransaction() + switch finish { + case "discard": try await transaction!.discard() + case "deallocate": transaction = nil + default: + try await transaction!.writeText("/a", "committed") + #expect(try await transaction!.commit().applied) + } + #expect(allocation.live == 1) + #expect(try await sibling.readData(from: "/a") == Data(repeating: 42, count: 4096)) + try await sibling.discard() + #expect(allocation.live == 0) + // Keep closed handles and the parent alive: neither may retain the payload. + withExtendedLifetime((transaction, sibling, ws)) {} + } + + @Test + func `cold base retains at most its budget and freezes overflow files`() async throws { + let storage = try TestSupport.temporaryDirectory("ColdBaseOverflow") + defer { TestSupport.remove(storage) } + let fs = InMemoryFileSystem() + let budget = TransactionBacking.memoryBudgetBytes + try await fs.writeFile(path: "/a", data: Data(count: budget), append: false) + try await fs.writeFile(path: "/z", data: Data("overflow".utf8), append: false) + let ws = Workspace(fileSystem: fs, persistence: .directory(storage)) + let transaction = try await ws.beginTransaction() + let retained = await ws.coldTransactionBase?.tree.nodes.values.reduce(0) { $0 + ($1.fileData?.count ?? 0) } + #expect(retained == budget) + try await fs.writeFile(path: "/z", data: Data("external".utf8), append: false) + #expect(try await transaction.readText("/z") == "overflow") + try await transaction.discard() + #expect(await ws.coldTransactionBase == nil) + } + + @Test(arguments: CoarseFileSystem.Mode.allCases) + func `same metadata parent writes conflict instead of being overwritten`(mode: CoarseFileSystem.Mode) async throws { + let fs = CoarseFileSystem(mode: mode) + let ws = Workspace(fileSystem: fs, checkpointCapture: mode == .verified ? .verifyContents : .trustMetadata) + try await ws.writeText("/a", "aaaa") + let tx = try await ws.beginTransaction() + try await fs.base.writeFile(path: "/a", data: Data("bbbb".utf8), append: false) + try await tx.writeText("/a", "cccc") + let commit = try await tx.commit() + #expect(commit.conflicts.map(\.kind) == [.bothModified]) + #expect(try await ws.readText("/a") == "bbbb") + } + + @Test(arguments: CoarseFileSystem.Mode.allCases) + func `unrelated draft edits checkpoint the actual concurrent parent contents`(mode: CoarseFileSystem.Mode) async throws { + let fs = CoarseFileSystem(mode: mode) + let ws = Workspace(fileSystem: fs, checkpointCapture: mode == .verified ? .verifyContents : .trustMetadata) + try await ws.writeText("/a", "aaaa") + let tx = try await ws.beginTransaction() + try await fs.base.writeFile(path: "/a", data: Data("bbbb".utf8), append: false) + try await tx.writeText("/b", "draft") + let commit = try await tx.commit() + let cp = try #require(commit.checkpoint) + let stored = try await ws.readText("/a", at: .checkpoint(cp.id)) + #expect(stored == "bbbb") + } + + @Test func `draft hard link append survives commit`() async throws { + let ws = Workspace() + try await ws.writeText("/a", "x") + let tx = try await ws.beginTransaction() + try await tx.createHardLink("/b", target: "/a") + try await tx.appendText("/a", "yy") + let draftB = try await tx.readText("/b") + _ = try await tx.commit() + let b = try await ws.readText("/b") + #expect(b == draftB) + } + + @Test(arguments: [(CoarseFileSystem.Mode.coarse, false), (.coarse, true), (.racy, true), (.verified, false)]) + func `path and rescan signals verify cached metadata`(scenario: (CoarseFileSystem.Mode, Bool)) async throws { + let (mode, rescan) = scenario + let fs = CoarseFileSystem(mode: mode) + try await fs.base.writeFile(path: "/a", data: Data("old\n".utf8), append: false) + let ws = Workspace(fileSystem: fs) + let stream = await ws.events(.init(includeCheckpoints: false)) + try await fs.base.writeFile(path: "/a", data: Data("new\n".utf8), append: false) + fs.signal(rescan ? .rescan : .paths(["/a": .entry])) + let got = await first(stream) + guard case let .changes(changes)? = got else { Issue.record("missing rewrite event"); return } + #expect(changes.touchedPaths == ["/a"]) + #expect(changes.changes.first?.diff != nil) + } + + @Test func `workspace retries a failed path refresh and preserves later signals`() async throws { + let fs = FailingSignalFS() + try await fs.base.writeFile(path: "/a", data: Data("old\n".utf8), append: false) + try await fs.base.writeFile(path: "/z", data: Data("z\n".utf8), append: false) + let ws = Workspace(fileSystem: fs) + let stream = await ws.events(.init(includeCheckpoints: false)) + try await fs.base.writeFile(path: "/a", data: Data("newer\n".utf8), append: false) + fs.failNext() + fs.signal(.paths(["/a": .entry])) + try await Task.sleep(for: .milliseconds(200)) + try await fs.base.writeFile(path: "/z", data: Data("zz\n".utf8), append: false) + fs.signal(.paths(["/z": .entry])) + var paths: Set = [] + while paths.count < 2, let e = await first(stream) { + if case let .changes(c) = e { paths.formUnion(c.touchedPaths) } + } + while let e = await first(stream, timeout: .milliseconds(400)) { + if case let .changes(c) = e { paths.formUnion(c.touchedPaths) } + } + #expect(paths == ["/a", "/z"]) + } + + @Test func `directory commits release accumulated file bytes and older base pins`() async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let ws = Workspace(persistence: .directory(storage)) + for i in 0..<20 { try await ws.writeText(WorkspacePath(normalizing: "/f\(i)"), "v") } + _ = try await ws.createCheckpoint() + for i in 0..<20 { + let tx = try await ws.beginTransaction() + try await tx.writeText(WorkspacePath(normalizing: "/f\(i)"), "changed \(i)") + _ = try await tx.commit() + } + let held = await ws.transactionBase?.nodes.values.filter { $0.fileData != nil }.count ?? -1 + #expect(held == 0) + let latest = try #require(try await ws.checkpoints().last) + #expect(TransactionRevisionPins.shared.checkpoints(in: ws.workspaceID) == [latest.id]) + } + + @Test func `later begins verify a reused base against coarse parent metadata`() async throws { + let fs = CoarseFileSystem() + let ws = Workspace(fileSystem: fs) + try await ws.writeText("/a", "aaaa") + try await ws.beginTransaction().discard() + try await fs.base.writeFile(path: "/a", data: Data("bbbb".utf8), append: false) + let tx = try await ws.beginTransaction() + #expect(try await tx.readText("/a") == "bbbb") + try await tx.discard() + } + + @Test func `transaction overwrites retain executable permissions`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/tool", data: Data("old".utf8), append: false) + try await local.setPermissions(path: "/tool", permissions: POSIXPermissions(0o755)) + let ws = Workspace(fileSystem: local) + let tx = try await ws.beginTransaction() + try await tx.writeText("/tool", "new") + #expect(try await tx.info("/tool").permissions == POSIXPermissions(0o755)) + _ = try await tx.commit() + #expect(try await local.stat(path: "/tool").permissions == POSIXPermissions(0o755)) + } + + #if os(macOS) + @Test func `FSEvents preserves scoped paths beneath private temporary roots`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + // Subscribe before writing: the stream itself, not Workspace, must preserve paths. + let stream = try await fs.changeSignals() + try await fs.writeFile(path: "/x", data: Data("changed".utf8), append: false) + let signal = await withTaskGroup(of: FileSystemChangeSignal?.self) { group in + group.addTask { + var combined: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + for await signal in stream { + guard case let .paths(paths) = signal else { return signal } + combined.merge(paths, uniquingKeysWith: max) + if combined["/x"] != nil { return .paths(combined) } + } + return nil + } + group.addTask { try? await Task.sleep(for: .seconds(2)); return nil } + let value = await group.next() ?? nil + group.cancelAll() + return value + } + guard case let .paths(paths)? = signal else { + Issue.record("expected scoped FSEvents notification, got \(String(describing: signal))") + return + } + #expect(paths["/x"] == .entry) + #expect(paths[.root] == nil || paths[.root] == .shallow) + #expect(!paths.values.contains(.recursive)) + #expect(!paths.keys.contains { LocalAtomicTemporaryName.matches($0.name) }) + } + + @Test func `FSEvents root created immediately before observation stays shallow after one write`() async throws { + let root = try TestSupport.temporaryDirectory("FreshRootWatch") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + // No settling delay: the root's own creation can arrive with the file event. + let stream = try await fs.changeSignals() + try await fs.writeFile(path: "/x", data: Data("changed".utf8), append: false) + let collector = Task { + var combined: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + var rescanned = false + for await signal in stream { + switch signal { + case .rescan: rescanned = true + case let .paths(paths): combined.merge(paths, uniquingKeysWith: max) + } + } + return (combined, rescanned) + } + defer { collector.cancel() } + // Unlike the first-event probe above, include delayed batches after /x. + try await Task.sleep(for: .seconds(2)) + collector.cancel() + let (paths, rescanned) = await collector.value + #expect(!rescanned) + #expect(paths["/x"] == .entry) + #expect(paths[.root] == nil || paths[.root] == .shallow) + #expect(!paths.values.contains(.recursive)) + } + #endif + + private func first(_ stream: AsyncStream, timeout: Duration = .seconds(5)) async -> WorkspaceEvent? { + await withTaskGroup(of: WorkspaceEvent?.self) { group in + group.addTask { var it = stream.makeAsyncIterator(); return await it.next() } + group.addTask { try? await Task.sleep(for: timeout); return nil } + let v = await group.next() ?? nil + group.cancelAll() + return v + } + } +} + +/// Tracks the actual buffer returned by the first filesystem read, independently +/// of transaction representation and allocator RSS reuse. +private final class TrackedColdAllocation: @unchecked Sendable { + private let lock = NSLock() + private var issued = false + private var retained = 0 + var live: Int { lock.withLock { retained } } + func wrapFirst(_ data: Data) -> Data { + let track = lock.withLock { + guard !issued else { return false } + issued = true + retained += 1 + return true + } + guard track else { return data } + let pointer = UnsafeMutableRawPointer.allocate(byteCount: data.count, alignment: 8) + data.copyBytes(to: pointer.assumingMemoryBound(to: UInt8.self), count: data.count) + return Data(bytesNoCopy: pointer, count: data.count, deallocator: .custom { pointer, _ in + pointer.deallocate() + self.lock.withLock { self.retained -= 1 } + }) + } +} + +final class FailingSignalFS: FileSystem, FileSystemChangeSource, @unchecked Sendable { + let base = InMemoryFileSystem() + let stream: AsyncStream + let continuation: AsyncStream.Continuation + let lock = NSLock() + var fail = false + init() { (stream, continuation) = AsyncStream.makeStream() } + func failNext() { lock.withLock { fail = true } } + func signal(_ s: FileSystemChangeSignal) { continuation.yield(s) } + func changeSignals() async throws -> AsyncStream { stream } + func changes() async throws -> AsyncStream { unscopedChanges(stream) } + func stat(path: WorkspacePath) async throws -> FileInfo { + if lock.withLock({ let f = fail; fail = false; return f }) { throw TestFileSystemError.injected } + return try await base.stat(path: path) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func readFile(path: WorkspacePath) async throws -> Data { + return try await base.readFile(path: path) + } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { try await base.glob(pattern: pattern, currentDirectory: currentDirectory) } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await base.writeFile(path: path, data: data, append: append) } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await base.remove(path: path, recursive: recursive) } + func move(from s: WorkspacePath, to d: WorkspacePath) async throws { try await base.move(from: s, to: d) } + func copy(from s: WorkspacePath, to d: WorkspacePath, recursive: Bool) async throws { try await base.copy(from: s, to: d, recursive: recursive) } +} diff --git a/Tests/WorkspaceTests/EditAtomicityTests.swift b/Tests/WorkspaceTests/EditAtomicityTests.swift index 8ddffe6..4b5ade3 100644 --- a/Tests/WorkspaceTests/EditAtomicityTests.swift +++ b/Tests/WorkspaceTests/EditAtomicityTests.swift @@ -208,7 +208,8 @@ struct EditAtomicityTests { func `checkpoint persistence failure retracts history and restores filesystem`() async throws { let storage = try TestSupport.temporaryDirectory() defer { TestSupport.remove(storage) } - let workspace = Workspace(persistence: .directory(storage)) + let fs = HookableFileSystem() + let workspace = Workspace(fileSystem: fs, persistence: .directory(storage)) try await workspace.writeText("/a", "original") let checkpoint = try await workspace.createCheckpoint() let transaction = try await workspace.beginTransaction() @@ -216,7 +217,9 @@ struct EditAtomicityTests { let before = try await workspace.history() let manifest = storage.appendingPathComponent(workspace.workspaceID.uuidString) .appendingPathComponent("snapshots").appendingPathComponent("\(checkpoint.snapshotId.uuidString).json") - try FileManager.default.removeItem(at: manifest) + // Remove the manifest only after merge reads, when commit starts writing. + // This still targets persistence compensation rather than lazy base loading. + await fs.onWrite { _ in try FileManager.default.removeItem(at: manifest) } await #expect(throws: (any Error).self) { try await transaction.commit() } #expect(try await workspace.readText("/a") == "original") #expect(try await workspace.history() == before) diff --git a/Tests/WorkspaceTests/EditPathTests.swift b/Tests/WorkspaceTests/EditPathTests.swift index 9c74016..0909c47 100644 --- a/Tests/WorkspaceTests/EditPathTests.swift +++ b/Tests/WorkspaceTests/EditPathTests.swift @@ -4,6 +4,46 @@ import Testing @Suite("Edit path coverage") struct EditPathTests { + static var localWrappers: [String] { + var wrappers = ["limited", "sandbox", "mounted", "authorized"] + #if canImport(Darwin) + wrappers.append("securityScoped") + #endif + return wrappers + } + + @Test(arguments: localWrappers) + func `wrapped local preview preserves overwrite modes and symlink semantics`(wrapper: String) async throws { + let root = try TestSupport.temporaryDirectory("WrappedPreview") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/tool", data: Data("old".utf8), append: false) + try await local.setPermissions(path: "/tool", permissions: POSIXPermissions(0o755)) + try await local.createSymlink(path: "/alias", target: "tool") + let fs: any FileSystem + switch wrapper { + case "limited": fs = LimitedFileSystem(base: local, limits: .init()) + case "sandbox": fs = try SandboxFileSystem(root: .url(root)) + case "mounted": fs = MountedFileSystem(mounts: [.init(mountPoint: "/mount", fileSystem: local)]) + #if canImport(Darwin) + case "securityScoped": fs = try SecurityScopedFileSystem(url: root) + #endif + default: fs = AuthorizedFileSystem(base: local, authorizer: PermissionAuthorizer { _ in .allow }) + } + let prefix = wrapper == "mounted" ? "/mount" : "" + let workspace = Workspace(fileSystem: fs) + let edits: [Edit] = [.writeText(try WorkspacePath(prefix + "/alias"), "new")] + let before = try await Snapshot.capture(from: fs) + let preview = try await workspace.preview(edits) + #expect(try await Snapshot.capture(from: fs).entry == before.entry) + let applied = try await workspace.apply(edits) + #expect(preview == applied.changes) + #expect(preview.touchedPaths == [try WorkspacePath(prefix + "/tool")]) + #expect(try await local.stat(path: "/tool").permissions == POSIXPermissions(0o755)) + #expect(try await local.readSymlink(path: "/alias") == "tool") + #expect(try await local.readFile(path: "/tool") == Data("new".utf8)) + } + enum Backend: CaseIterable, Sendable { case memory, local, mounted } struct Scenario: Sendable, CustomStringConvertible { @@ -14,6 +54,7 @@ struct EditPathTests { static let scenarios: [Scenario] = [ .init(description: "nested mount write", edits: [.writeText("/mnt/data/x", "hi")]), + .init(description: "overwrite mode", edits: [.setPermissions("/dir/a", POSIXPermissions(0o600)), .writeText("/dir/a", "new")]), .init(description: "writeText", edits: [.writeText("/dir/a", "new")]), .init(description: "appendText", edits: [.appendText("/dir/a", "new")]), .init(description: "writeData", edits: [.writeData("/dir/a", Data([1, 2]))]), @@ -91,6 +132,7 @@ struct EditPathTests { let before = try await Snapshot.capture(from: fs) let edits = scenario.edits + (failing ? [.copy(from: "/absent", to: "/failure")] : []) if failing || (backend != .local && scenario.memoryRejects) { + await #expect(throws: (any Error).self) { try await workspace.preview(edits) } do { try await workspace.apply(edits) Issue.record("expected edit failure") @@ -101,7 +143,10 @@ struct EditPathTests { #expect(after.entry == before.entry) #expect(try await workspace.history().isEmpty) } else { + let preview = try await workspace.preview(edits) + #expect(try await Snapshot.capture(from: fs).entry == before.entry) let result = try await workspace.apply(edits) + #expect(preview == result.changes) let after = try await Snapshot.capture(from: fs) let expected = ChangeSet.compare(before: before, after: after, maxTextBytes: DiffOptions.default.maxTextBytes) #expect(result.changes == expected) diff --git a/Tests/WorkspaceTests/FileSystemTraversalTests.swift b/Tests/WorkspaceTests/FileSystemTraversalTests.swift index f0e7e31..97afcd0 100644 --- a/Tests/WorkspaceTests/FileSystemTraversalTests.swift +++ b/Tests/WorkspaceTests/FileSystemTraversalTests.swift @@ -8,6 +8,30 @@ import Glibc #endif struct FileSystemTraversalTests { + @Test + func mountedWalkRebasesPathsAndPreservesShadowing() async throws { + let root = try TestSupport.temporaryDirectory("MountedWalk") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/base", data: Data("base".utf8), append: false) + try await local.writeFile(path: "/mount/hidden", data: Data("hidden".utf8), append: false) + let counted = HookableFileSystem(base: local, scopedWalk: true) + let mounted = MountedFileSystem(base: counted, mounts: [ + .init(mountPoint: "/mount", fileSystem: InMemoryFileSystem()), + .init(mountPoint: "/synthetic/deep", fileSystem: counted), + ]) + try await mounted.writeFile(path: "/mount/visible", data: Data("visible".utf8), append: false) + let snapshot = try await Snapshot.capture(from: mounted) + #expect(await counted.walks.contains(.root)) + #expect(!(await counted.walks.contains("/mount"))) + let reference = try await Snapshot.capture(from: HookableFileSystem(base: mounted)) + #expect(snapshot.entry == reference.entry) + let metadata = try await MetadataIndex.capture(from: mounted) + #expect(metadata.entries["/mount/hidden"] == nil) + #expect(metadata.entries["/mount/visible"]?.info.kind == .file) + #expect(metadata.entries["/synthetic/deep/mount/hidden"]?.info.kind == .file) + } + @Test func deepCaptureBoundsDescriptors() async throws { // Resource limits are process-wide. Re-execute only this test in a child so @@ -54,7 +78,7 @@ struct FileSystemTraversalTests { let local = try LocalFileSystem(root: root) try await local.writeFile(path: path, data: Data("leaf".utf8), append: false) let captured = try await Snapshot.capture(from: local) - let reference = try await Snapshot.capture(from: MountedFileSystem(base: local)) + let reference = try await Snapshot.capture(from: HookableFileSystem(base: local)) #expect(captured.entry == reference.entry) var visited = false try await local.walk { info, read, _ in @@ -99,8 +123,8 @@ struct FileSystemTraversalTests { try FileManager.default.createSymbolicLink(atPath: root.appendingPathComponent(name).path, withDestinationPath: target) } - // Mounted with no mounts preserves the namespace and deliberately uses per-path capture. - let perPath = MountedFileSystem(base: local) + // A plain forwarding test backend deliberately keeps the per-path reference walk. + let perPath = HookableFileSystem(base: local) for path: WorkspacePath in [.root, "/dir", "/dir/file", "/dangling", "/cycle", "/escape", "/absent", "/alias/empty"] { let bulk = try await Snapshot.capture(from: local, at: path) let reference = try await Snapshot.capture(from: perPath, at: path) @@ -147,7 +171,7 @@ struct FileSystemTraversalTests { try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("alias"), withDestinationURL: root.appendingPathComponent("private").appendingPathComponent(name)) #expect(try await local.readFile(path: "/alias") == Data("inside".utf8)) - #expect(try await Snapshot.capture(from: local).entry == Snapshot.capture(from: MountedFileSystem(base: local)).entry) + #expect(try await Snapshot.capture(from: local).entry == Snapshot.capture(from: HookableFileSystem(base: local)).entry) let authorized = AuthorizedFileSystem(base: local, authorizer: RuleBasedPermissionAuthorizer( rules: [.init(operations: [.readFile], pathPrefix: "/private", effect: .deny(message: "private"))], defaultEffect: .allow)) @@ -215,7 +239,7 @@ struct FileSystemTraversalTests { let local = try LocalFileSystem(root: root) try await local.writeFile(path: "/blocked/child", data: Data(), append: false) #expect(chmod(blocked.path, 0) == 0) - for fs: any FileSystem in [local, MountedFileSystem(base: local)] { + for fs: any FileSystem in [local, HookableFileSystem(base: local)] { do { _ = try await Snapshot.capture(from: fs) Issue.record("capture must fail when a directory cannot be enumerated") @@ -345,7 +369,7 @@ struct FileSystemTraversalTests { let request = SearchRequest(pattern: .literal("match", caseSensitive: true), files: selection, limits: .init(maxFileBytes: 100, maxFiles: 3, maxMatches: 2), matchOffset: 1) let fast = try await Workspace.search(request, on: local) - let fallback = try await Workspace.search(request, on: MountedFileSystem(base: local)) + let fallback = try await Workspace.search(request, on: HookableFileSystem(base: local)) #expect(fast == fallback) #expect(fast.totalMatches > 0) if selection.include != ["alias/file"] { diff --git a/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift b/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift index cda6b1d..a6d8eae 100644 --- a/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift +++ b/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift @@ -6,6 +6,132 @@ import Testing @Suite("Linux recursive filesystem watching") struct LinuxFileChangeStreamTests { + @Test(arguments: ["limited", "sandbox", "mounted", "authorized"]) + func `wrappers deliver scoped native changes in their virtual namespace`(wrapper: String) async throws { + let root = try TestSupport.temporaryDirectory("WrappedWatch") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let source: any FileSystemChangeSource = switch wrapper { + case "limited": LimitedFileSystem(base: local, limits: .init()) + case "sandbox": try SandboxFileSystem(root: .url(root)) + case "mounted": MountedFileSystem(mounts: [.init(mountPoint: "/mount", fileSystem: local)]) + default: AuthorizedFileSystem(base: local, authorizer: PermissionAuthorizer { _ in .allow }) + } + let stream = try await source.changeSignals() + let signals = LinuxScopedSignals() + let consumer = Task { for await signal in stream { await signals.record(signal) } } + defer { consumer.cancel() } + // Mounted subscriptions are installed asynchronously and announce their + // initial reconciliation after registering all native streams. + if wrapper == "mounted" { #expect(await signals.waitForRescan()) } + try await local.writeFile(path: "/x", data: Data("new".utf8), append: false) + let path: WorkspacePath = wrapper == "mounted" ? "/mount/x" : "/x" + #expect(await signals.waitForPath(path, after: 0)) + #expect(await signals.scopes[path] == .entry) + if let mounted = source as? MountedFileSystem { + let rescans = await signals.rescans + mounted.mount("/added", fileSystem: local) + #expect(await signals.waitForRescan(after: rescans)) + let previous = await signals.count + try await local.writeFile(path: "/later", data: Data("later".utf8), append: false) + #expect(await signals.waitForPath("/added/later", after: previous)) + #expect(await signals.waitForPath("/mount/later", after: previous)) + } + } + + @Test + func `native stored spellings retain canonical twins and combining mark components`() async throws { + let container = try TestSupport.temporaryDirectory("UnicodeWatch") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("\u{301}root") + let fs = try LocalFileSystem(root: root) + let paths: [WorkspacePath] = ["/caf\u{e9}/\u{301}file", "/cafe\u{301}/\u{301}file"] + for path in paths { try await fs.writeFile(path: path, data: Data("old".utf8), append: false) } + let stream = try await fs.changeSignals() + let signals = LinuxScopedSignals() + let consumer = Task { for await signal in stream { await signals.record(signal) } } + defer { consumer.cancel() } + for path in paths { + try await fs.writeFile(path: path, data: Data("new".utf8), append: false) + #expect(await signals.waitForPath(path, after: 0)) + #expect(await signals.scopes[path] == .entry) + } + #expect(await signals.rescans == 0) + } + + @Test + func `directory creation and removal emit recursive path invalidations`() async throws { + let root = try TestSupport.temporaryDirectory("ScopedDirectoryWatch") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + let stream = try await fs.changeSignals() + let signals = LinuxScopedSignals() + let consumer = Task { for await signal in stream { await signals.record(signal) } } + defer { consumer.cancel() } + try await fs.createDirectory(path: "/child", recursive: false) + #expect(await signals.waitForPath("/child", after: 0)) + #expect(await signals.scopes["/child"] == .recursive) + let beforeRemoval = await signals.count + try await fs.remove(path: "/child", recursive: true) + #expect(await signals.waitForPath("/child", after: beforeRemoval)) + // Include any queued IN_IGNORED notification after watch retirement. + try await Task.sleep(for: .milliseconds(150)) + #expect(await signals.rescans == 0) + } + + @Test + func `file writes are entry scoped and directory metadata is shallow`() async throws { + let root = try TestSupport.temporaryDirectory("ScopedFileWatch") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + try await fs.createDirectory(path: "/d", recursive: false) + let stream = try await fs.changeSignals() + let signals = LinuxScopedSignals() + let consumer = Task { for await signal in stream { await signals.record(signal) } } + defer { consumer.cancel() } + try await fs.writeFile(path: "/x", data: Data("changed".utf8), append: false) + #expect(await signals.waitForPath("/x", after: 0)) + try await fs.setPermissions(path: "/d", permissions: POSIXPermissions(0o700)) + #expect(await signals.waitForPath("/d", after: 0)) + let userName = ".workspace-" + UUID().uuidString + let userPath = try WorkspacePath("/" + userName) + try await fs.writeFile(path: userPath, data: Data("user file".utf8), append: false) + #expect(await signals.waitForPath(userPath, after: 0)) + try await Task.sleep(for: .milliseconds(150)) + #expect(await signals.rescans == 0) + #expect(await signals.scopes["/x"] == .entry) + #expect(await signals.scopes["/d"] == .shallow) + #expect(await signals.scopes[userPath] == .entry) + #expect(await !signals.scopes.keys.contains { LocalAtomicTemporaryName.matches($0.name) }) + } + + @Test(arguments: [false, true]) + func `directory rename and move out do not rescan after retiring watches`(moveOut: Bool) async throws { + let container = try TestSupport.temporaryDirectory("RetiredWatch") + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + try FileManager.default.createDirectory(at: root.appendingPathComponent("d/sub"), withIntermediateDirectories: true) + let fs = try LocalFileSystem(root: root) + let stream = try await fs.changeSignals() + let signals = LinuxScopedSignals() + let consumer = Task { for await signal in stream { await signals.record(signal) } } + defer { consumer.cancel() } + try FileManager.default.moveItem(at: root.appendingPathComponent("d"), + to: (moveOut ? container : root).appendingPathComponent("e")) + #expect(await signals.waitForPath("/d", after: 0)) + if !moveOut { #expect(await signals.waitForPath("/e", after: 0)) } + // IN_IGNORED from our rm_watch arrives in a later batch. + try await Task.sleep(for: .milliseconds(400)) + #expect(await signals.rescans == 0) + #expect(await signals.scopes["/d"] == .recursive) + if !moveOut { + #expect(await signals.scopes["/e"] == .recursive) + let previous = await signals.count + try await fs.writeFile(path: "/e/sub/new", data: Data("new".utf8), append: false) + #expect(await signals.waitForPath("/e/sub/new", after: previous)) + } + } + @Test func `recursive watches observe writes inside hidden directories`() async throws { let root = try TestSupport.temporaryDirectory("HiddenDirectoryWatch") @@ -144,6 +270,40 @@ struct LinuxFileChangeStreamTests { } } +private actor LinuxScopedSignals { + var count = 0 + var rescans = 0 + var scopes: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + var lastSeen: [WorkspacePath: Int] = [:] + func waitForRescan(after previous: Int = 0) async -> Bool { + let deadline = ContinuousClock.now.advanced(by: .seconds(2)) + while ContinuousClock.now < deadline { + if rescans > previous { return true } + try? await Task.sleep(for: .milliseconds(10)) + } + return false + } + func record(_ signal: FileSystemChangeSignal) { + count += 1 + switch signal { + case .rescan: rescans += 1 + case let .paths(paths): + for (path, scope) in paths { + lastSeen[path] = count + scopes[path] = max(scopes[path] ?? .entry, scope) + } + } + } + func waitForPath(_ path: WorkspacePath, after previous: Int) async -> Bool { + let deadline = ContinuousClock.now.advanced(by: .seconds(2)) + while ContinuousClock.now < deadline { + if (lastSeen[path] ?? 0) > previous { return true } + try? await Task.sleep(for: .milliseconds(10)) + } + return false + } +} + private actor LinuxInvalidationCounter { private(set) var value = 0 func increment() { value += 1 } diff --git a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift index 0d631a3..cd75648 100644 --- a/Tests/WorkspaceTests/PathIdentityRegressionTests.swift +++ b/Tests/WorkspaceTests/PathIdentityRegressionTests.swift @@ -221,6 +221,44 @@ struct PathIdentityRegressionTests { } } + @Test + func transactionMergesCanonicalTwinsIndependently() async throws { + let workspace = Workspace() + try await workspace.createDirectory(nfc) + try await workspace.createDirectory(nfd) + try await workspace.writeText(nfc.appending("\u{301}file"), "NFC") + try await workspace.writeText(nfd.appending("\u{301}file"), "NFD") + let transaction = try await workspace.beginTransaction() + try await transaction.writeText(nfc.appending("\u{301}file"), "draft") + try await workspace.writeText(nfd.appending("\u{301}file"), "parent") + #expect(try await transaction.preview().touchedPaths == [nfc.appending("\u{301}file")]) + #expect(try await transaction.commit().applied) + #expect(try await workspace.readText(nfc.appending("\u{301}file")) == "draft") + #expect(try await workspace.readText(nfd.appending("\u{301}file")) == "parent") + } + + @Test + func observationDetectsByteOnlySymlinkChangesWithIdenticalMetadata() async throws { + let fs = CoarseFileSystem() + // Canonically equivalent targets with equal UTF-8 lengths and fixed metadata. + // Only the stored target bytes distinguish the two observations. + try await fs.base.createSymlink(path: "/link", target: "a\u{301}\u{323}") + var observation = try await FilesystemObservation.start(on: fs, budget: 0) + try await fs.base.remove(path: "/link", recursive: false) + try await fs.base.createSymlink(path: "/link", target: "a\u{323}\u{301}") + #expect(try await observation.refresh(on: fs, signal: .rescan).touchedPaths == ["/link"]) + } + + @Test + func nativePathRebasingUsesStoredByteComponents() throws { + let root = "/private/cafe\u{301}/\u{301}root" + #expect(nativeChangePath(root + "/\u{301}file", relativeTo: root) == "/\u{301}file") + #expect(nativeChangePath(root, relativeTo: root) == .root) + #expect(nativeChangePath("/caf\u{e9}/child", relativeTo: "/cafe\u{301}") == nil) + #expect(nativeChangePath("/root-other/file", relativeTo: "/root") == nil) + #expect(nativeChangePath("/\u{301}file", relativeTo: "/") == "/\u{301}file") + } + #if canImport(Darwin) private func caseInsensitive(_ root: URL) throws -> Bool { try seed(root, "case-probe") diff --git a/Tests/WorkspaceTests/ReviewRevisionTests.swift b/Tests/WorkspaceTests/ReviewRevisionTests.swift new file mode 100644 index 0000000..5fa8669 --- /dev/null +++ b/Tests/WorkspaceTests/ReviewRevisionTests.swift @@ -0,0 +1,147 @@ +import Foundation +import Testing +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif +@testable import Workspace + +@Suite("Review revision regressions") +struct ReviewRevisionTests { + @Test func `unreadable content reports an omission without blocking later events or retrying`() async throws { + let fs = SignallingFileSystem() + let ws = Workspace(fileSystem: fs) + let events = await ws.events(.init(includeCheckpoints: false)) + try await fs.writeFile(path: "/bad", data: Data("secret".utf8), append: false) + await fs.base.denyReads(at: "/bad") + fs.signal(.paths(["/bad": .entry])) + guard case let .changes(bad)? = await first(events) else { Issue.record("missing unreadable entry"); return } + #expect(bad.touchedPaths == ["/bad"]) + #expect(bad.textDiffOmissions == [.init(path: "/bad", reason: .contentReadFailed)]) + try await fs.writeFile(path: "/good", data: Data("good".utf8), append: false) + fs.signal(.paths(["/good": .entry])) + guard case let .changes(good)? = await first(events) else { Issue.record("later change blocked"); return } + #expect(good.touchedPaths == ["/good"]) + try await Task.sleep(for: .milliseconds(400)) + #expect(await fs.base.reads.filter { $0 == "/bad" }.count == 1) + } + + #if os(macOS) || os(Linux) + @Test func `native unreadable files do not block later filesystem events`() async throws { + // Root can read mode-000 files; the Docker unprivileged suite exercises this case. + guard geteuid() != 0 else { return } + let root = try TestSupport.temporaryDirectory("UnreadableObservation") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let ws = Workspace(fileSystem: local) + let events = await ws.events(.init(includeCheckpoints: false)) + try await local.writeFile(path: "/bad", data: Data("secret".utf8), append: false) + try await local.setPermissions(path: "/bad", permissions: POSIXPermissions(0)) + guard case let .changes(bad)? = await first(events) else { Issue.record("missing unreadable native event"); return } + #expect(bad.textDiffOmissions == [.init(path: "/bad", reason: .contentReadFailed)]) + try await local.writeFile(path: "/good", data: Data("good".utf8), append: false) + guard case let .changes(good)? = await first(events) else { Issue.record("later native event blocked"); return } + #expect(good.touchedPaths == ["/good"]) + } + #endif + + @Test func `metadata retries stop after three attempts and a native signal restarts retained work`() async throws { + let fs = SignallingFileSystem() + let ws = Workspace(fileSystem: fs) + let events = await ws.events(.init(includeCheckpoints: false)) + try await fs.writeFile(path: "/a", data: Data("a".utf8), append: false) + let stats = await fs.base.statCount + await fs.base.failStat(NSError(domain: NSPOSIXErrorDomain, code: Int(EACCES))) + fs.signal(.rescan) + let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + while await fs.base.statCount - stats < 3, ContinuousClock.now < deadline { + try await Task.sleep(for: .milliseconds(10)) + } + #expect(await fs.base.statCount - stats == 3) + try await Task.sleep(for: .milliseconds(500)) + #expect(await fs.base.statCount - stats == 3) + await fs.base.failStat(nil) + try await fs.writeFile(path: "/z", data: Data("z".utf8), append: false) + fs.signal(.paths(["/z": .entry])) + guard case let .changes(changes)? = await first(events) else { Issue.record("pending work was not resumed"); return } + #expect(changes.touchedPaths == ["/a", "/z"]) + } + + @Test(arguments: [false, true]) + func `compaction preserves reusable bases on dry runs and while their checkpoint survives`(directory: Bool) async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let fs = HookableFileSystem() + try await fs.writeFile(path: "/a", data: Data("frozen".utf8), append: false) + let ws = Workspace(fileSystem: fs, persistence: directory ? .directory(storage) : .memory) + try await ws.beginTransaction().discard() + let checkpoint = try await ws.createCheckpoint() + // Simulate a newer revision arriving through the store without re-seeding this instance. + _ = try await ws.store.saveRevision(Snapshot.capture(from: fs.base), draft: .init( + workspaceID: checkpoint.workspaceId, label: nil, preferredParentID: checkpoint.id, origin: .manual)) + let reads = await fs.reads.count + _ = try await ws.compact(retaining: .latest(1, preservingLabeled: false), dryRun: true) + try await ws.beginTransaction().discard() + #expect(await fs.reads.count == reads) + _ = try await ws.compact(retaining: .latest(1, preservingLabeled: false, preserving: [checkpoint.id])) + try await ws.beginTransaction().discard() + #expect(await fs.reads.count == reads) + _ = try await ws.compact(retaining: .latest(1, preservingLabeled: false)) + #expect(await ws.transactionBase == nil) + try await ws.beginTransaction().discard() + #expect(await fs.reads.count == reads + 1) + } + + @Test func `old timestamps without identities can reuse contents`() async throws { + let source = CoarseFileSystem(timestamp: Date(timeIntervalSince1970: 1_000)) + let fs = HookableFileSystem(base: source) + try await fs.writeFile(path: "/a", data: Data("old".utf8), append: false) + let ws = Workspace(fileSystem: fs) + try await ws.beginTransaction().discard() + let reads = await fs.reads.count + try await ws.beginTransaction().discard() + #expect(await fs.reads.count == reads) + } + + @Test func `fresh in memory timestamps reuse unchanged contents and detect appends`() async throws { + let fs = HookableFileSystem() + try await fs.writeFile(path: "/a", data: Data("a".utf8), append: false) + let ws = Workspace(fileSystem: fs) + try await ws.beginTransaction().discard() + let reads = await fs.reads.count + let tx = try await ws.beginTransaction() + #expect(await fs.reads.count == reads) + try await fs.writeFile(path: "/a", data: Data("b".utf8), append: true) + try await tx.writeText("/a", "draft") + #expect(try await tx.commit().conflicts.map(\.path) == ["/a"]) + } + + @Test func `in memory draft directory and permission symlink semantics match apply`() async throws { + let source = InMemoryFileSystem() + try await source.createDirectory(path: "/target", recursive: false) + try await source.createSymlink(path: "/link", target: "/target") + let ws = Workspace(fileSystem: source) + let tx = try await ws.beginTransaction() + let edits: [Edit] = [.setPermissions("/link", POSIXPermissions(0o700))] + let expected = try await ws.preview(edits) + _ = try await tx.apply(edits) + #expect(try await tx.preview() == expected) + #expect(try await tx.commit().applied) + #expect(try await source.stat(path: "/link").permissions == POSIXPermissions(0o700)) + #expect(try await source.stat(path: "/target").permissions == POSIXPermissions(0o755)) + let mkdir = try await ws.beginTransaction() + await #expect(throws: (any Error).self) { try await mkdir.apply([.createDirectory("/link", recursive: true)]) } + await #expect(throws: (any Error).self) { try await ws.apply([.createDirectory("/link", recursive: true)]) } + } + + private func first(_ stream: AsyncStream) async -> WorkspaceEvent? { + await withTaskGroup(of: WorkspaceEvent?.self) { group in + group.addTask { var iterator = stream.makeAsyncIterator(); return await iterator.next() } + group.addTask { try? await Task.sleep(for: .seconds(2)); return nil } + let event = await group.next() ?? nil + group.cancelAll() + return event + } + } +} diff --git a/Tests/WorkspaceTests/ScopedObservationTests.swift b/Tests/WorkspaceTests/ScopedObservationTests.swift new file mode 100644 index 0000000..0b07b82 --- /dev/null +++ b/Tests/WorkspaceTests/ScopedObservationTests.swift @@ -0,0 +1,154 @@ +import Foundation +import Testing +#if os(macOS) +import CoreServices +#endif +@testable import Workspace + +@Suite("Scoped filesystem observation") +struct ScopedObservationTests { + @Test func `shallow parent plus entry signals leave unrelated subtrees and contents alone`() async throws { + let fs = SignallingFileSystem() + try await fs.base.createDirectory(path: "/unrelated/deep", recursive: true) + try await fs.base.writeFile(path: "/unrelated/deep/a", data: Data("keep".utf8), append: false) + try await fs.base.writeFile(path: "/sibling", data: Data("keep".utf8), append: false) + try await fs.base.writeFile(path: "/x", data: Data("before".utf8), append: false) + let ws = Workspace(fileSystem: fs) + let events = await ws.events(.init(includeCheckpoints: false)) + let lists = await fs.base.listings.count + let reads = await fs.base.reads.count + try await fs.base.writeFile(path: "/x", data: Data("after".utf8), append: false) + // Separate signals exercise Workspace's burst coalescing as well as refresh. + fs.signal(.paths([.root: .shallow])) + fs.signal(.paths(["/x": .entry])) + guard case let .changes(change)? = await first(events) else { Issue.record("missing external event"); return } + #expect(change.touchedPaths == ["/x"]) + #expect(await fs.base.listings.dropFirst(lists).allSatisfy { $0 == .root }) + #expect(await fs.base.reads.dropFirst(reads).allSatisfy { $0 == "/x" }) + // The untouched descendants must remain in the observation index. + try await fs.base.remove(path: "/unrelated", recursive: true) + fs.signal(.paths([.root: .shallow])) + guard case let .changes(deleted)? = await first(events) else { Issue.record("missing deletion event"); return } + #expect(deleted.touchedPaths == ["/unrelated", "/unrelated/deep", "/unrelated/deep/a"]) + } + + @Test func `shallow listings capture added and replaced directories recursively`() async throws { + let fs = InMemoryFileSystem() + try await fs.createDirectory(path: "/d", recursive: false) + try await fs.writeFile(path: "/d/old", data: Data("old".utf8), append: false) + var observation = try await FilesystemObservation.start(on: fs, budget: 1024) + try await fs.remove(path: "/d", recursive: true) + try await fs.createDirectory(path: "/d/nested", recursive: true) + try await fs.writeFile(path: "/d/nested/new", data: Data("new".utf8), append: false) + try await fs.createDirectory(path: "/added", recursive: false) + try await fs.writeFile(path: "/added/file", data: Data("file".utf8), append: false) + let changes = try await observation.refresh(on: fs, signal: .paths([.root: .shallow])) + #expect(changes.touchedPaths == ["/added", "/added/file", "/d/nested", "/d/nested/new", "/d/old"]) + #expect(observation.metadata.entries["/d/old"] == nil) + #expect(observation.metadata.entries["/d/nested/new"] != nil) + } + + @Test func `atomic temporary names do not suppress similarly named user entries`() { + let generated = LocalAtomicTemporaryName.make() + #expect(LocalAtomicTemporaryName.matches(generated)) + for name in [".workspace-" + UUID().uuidString, ".workspace-atomic-" + UUID().uuidString + "-" + UUID().uuidString, + generated + ".txt", String(generated.dropLast()), generated.lowercased()] { + #expect(!LocalAtomicTemporaryName.matches(name)) + } + } + + #if os(macOS) + @Test func `FSEvents root item events are shallow only while the original directory remains`() throws { + let container = try TestSupport.temporaryDirectory("RootItemIdentity").resolvingSymlinksInPath() + defer { TestSupport.remove(container) } + let root = container.appendingPathComponent("root") + let moved = container.appendingPathComponent("moved") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + let mapper = FSEventsChangeStream.EventMapper(root: root.path) + let file = (path: root.path + "/x", flags: UInt32(kFSEventStreamEventFlagItemCreated | kFSEventStreamEventFlagItemIsFile)) + for flag in [kFSEventStreamEventFlagItemCreated, kFSEventStreamEventFlagItemRenamed, + kFSEventStreamEventFlagItemRemoved, kFSEventStreamEventFlagItemInodeMetaMod] { + let flags = UInt32(flag | kFSEventStreamEventFlagItemIsDir) + #expect(mapper.signal(for: [(root.path, flags), file]) == .paths([.root: .shallow, "/x": .entry])) + } + let createdDirectory = UInt32(kFSEventStreamEventFlagItemCreated | kFSEventStreamEventFlagItemIsDir) + #expect(mapper.signal(for: [(root.path + "/child", createdDirectory)]) == .paths(["/child": .recursive])) + for flag in [kFSEventStreamEventFlagRootChanged, kFSEventStreamEventFlagMustScanSubDirs, + kFSEventStreamEventFlagUserDropped, kFSEventStreamEventFlagKernelDropped] { + #expect(mapper.signal(for: [(root.path, createdDirectory | UInt32(flag)), file]) == .rescan) + } + + // Check the actual directory identity, even when only a child item arrives. + try FileManager.default.moveItem(at: root, to: moved) + #expect(mapper.signal(for: [file]) == .rescan) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + #expect(mapper.signal(for: [(root.path, createdDirectory), file]) == .rescan) + try FileManager.default.removeItem(at: root) + try FileManager.default.createSymbolicLink(at: root, withDestinationURL: moved) + #expect(mapper.signal(for: [file]) == .rescan) + } + + @Test func `FSEvents distinguishes item metadata from structural and dropped notifications`() { + let metadataFlags = [kFSEventStreamEventFlagItemModified, kFSEventStreamEventFlagItemInodeMetaMod, + kFSEventStreamEventFlagItemXattrMod, kFSEventStreamEventFlagItemFinderInfoMod, + kFSEventStreamEventFlagItemChangeOwner] + for flag in metadataFlags { + #expect(FSEventsChangeStream.scope(for: UInt32(flag | kFSEventStreamEventFlagItemIsDir)) == .shallow) + #expect(FSEventsChangeStream.scope(for: UInt32(flag | kFSEventStreamEventFlagItemIsFile)) == .entry) + } + for flag in [kFSEventStreamEventFlagItemCreated, kFSEventStreamEventFlagItemRemoved, kFSEventStreamEventFlagItemRenamed] { + #expect(FSEventsChangeStream.scope(for: UInt32(flag | kFSEventStreamEventFlagItemIsDir)) == .recursive) + #expect(FSEventsChangeStream.scope(for: UInt32(flag | kFSEventStreamEventFlagItemIsFile)) == .entry) + } + #expect(FSEventsChangeStream.scope(for: UInt32(kFSEventStreamEventFlagMustScanSubDirs)) == .recursive) + for flag in [kFSEventStreamEventFlagUserDropped, kFSEventStreamEventFlagKernelDropped, + kFSEventStreamEventFlagRootChanged, kFSEventStreamEventFlagEventIdsWrapped] { + #expect(FSEventsChangeStream.scope(for: UInt32(flag | kFSEventStreamEventFlagItemIsDir)) == nil) + } + #expect(FSEventsChangeStream.scope(for: 0) == nil) + } + + @Test func `one native write in a medium tree avoids a recursive metadata walk`() async throws { + let root = try TestSupport.temporaryDirectory("ScopedMediumWatch") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + for directory in 0..<32 { + try await local.createDirectory(path: WorkspacePath("/d\(directory)"), recursive: false) + for file in 0..<100 { + try await local.writeFile(path: WorkspacePath("/d\(directory)/f\(file)"), data: Data("keep".utf8), append: false) + } + } + try await local.writeFile(path: "/x", data: Data("before".utf8), append: false) + let fs = SignallingFileSystem(base: local, source: local) + let ws = Workspace(fileSystem: fs) + let events = await ws.events(.init(includeCheckpoints: false)) + // Drain fixture setup notifications before measuring the external write. + try await Task.sleep(for: .milliseconds(250)) + let lists = await fs.base.listings.count + let stats = await fs.base.statCount + let reads = await fs.base.reads.count + try await local.writeFile(path: "/x", data: Data("after".utf8), append: false) + guard case let .changes(change)? = await first(events, timeout: .seconds(3)) else { + Issue.record("missing native external event"); return + } + #expect(change.touchedPaths == ["/x"]) + // Include delayed batches for the same atomic replacement. + try await Task.sleep(for: .milliseconds(200)) + let refreshed = await Array(fs.base.listings.dropFirst(lists)) + #expect(refreshed.allSatisfy { $0 == .root }) + #expect(refreshed.count <= 4) + #expect(await fs.base.statCount - stats < 20) + #expect(await fs.base.reads.dropFirst(reads).allSatisfy { $0 == "/x" }) + } + #endif + + private func first(_ stream: AsyncStream, timeout: Duration = .seconds(5)) async -> WorkspaceEvent? { + await withTaskGroup(of: WorkspaceEvent?.self) { group in + group.addTask { var iterator = stream.makeAsyncIterator(); return await iterator.next() } + group.addTask { try? await Task.sleep(for: timeout); return nil } + let event = await group.next() ?? nil + group.cancelAll() + return event + } + } +} diff --git a/Tests/WorkspaceTests/SearchEquivalenceTests.swift b/Tests/WorkspaceTests/SearchEquivalenceTests.swift index b0a4b65..33e7285 100644 --- a/Tests/WorkspaceTests/SearchEquivalenceTests.swift +++ b/Tests/WorkspaceTests/SearchEquivalenceTests.swift @@ -14,7 +14,7 @@ struct SearchEquivalenceTests { for path in expected.reversed() { try await local.createFile(path: path, data: Data("match".utf8)) } - let fallback = MountedFileSystem(base: local) + let fallback = HookableFileSystem(base: local) for fs: any FileSystem in [local, fallback] { var paths: [WorkspacePath] = [] try await fs.walk { info, _, _ in diff --git a/Tests/WorkspaceTests/TestSupport.swift b/Tests/WorkspaceTests/TestSupport.swift index c6afc24..198c54c 100644 --- a/Tests/WorkspaceTests/TestSupport.swift +++ b/Tests/WorkspaceTests/TestSupport.swift @@ -16,43 +16,72 @@ enum TestSupport { enum TestFileSystemError: Error { case injected } -actor HookableFileSystem: FileSystem { +actor HookableFileSystem: FileSystem, ExactTimestampFileSystem, TraversingFileSystem { + var hasExactContentTimestamps: Bool { get async { await base.supportsExactContentTimestamps } } let base: any FileSystem var reads: [WorkspacePath] = [] + var statCount = 0 + var walks: [WorkspacePath] = [] + private let scopedWalk: Bool + nonisolated var supportsScopedWalk: Bool { scopedWalk && base.supportsScopedWalk } + nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + visit: FileSystemWalkVisitor) async throws { + guard supportsScopedWalk else { + return try await walkUsingPaths(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + await recordWalk(path) + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + } + private func recordWalk(_ path: WorkspacePath) { walks.append(path) } + func resetMetadataCounts() { walks = []; statCount = 0; listings = [] } + var listings: [WorkspacePath] = [] var writes: [WorkspacePath] = [] var permissionWrites: [(path: WorkspacePath, permissions: POSIXPermissions)] = [] private let forwardsPermissions: Bool + private let transformRead: (@Sendable (Data) -> Data)? private var writeHook: (@Sendable (WorkspacePath) async throws -> Void)? private var readHook: (@Sendable (WorkspacePath) async throws -> Void)? private var readFailure = false private var statError: NSError? - func failStat(_ error: NSError) { statError = error } + func failStat(_ error: NSError?) { statError = error } + var unreadable: Set = [] + func denyReads(at path: WorkspacePath) { unreadable.insert(path) } private var removeFailure = false func failRemovals() { removeFailure = true } - init(base: any FileSystem = InMemoryFileSystem(), forwardsPermissions: Bool = false) { + init(base: any FileSystem = InMemoryFileSystem(), forwardsPermissions: Bool = false, scopedWalk: Bool = false, + transformRead: (@Sendable (Data) -> Data)? = nil) { self.base = base self.forwardsPermissions = forwardsPermissions + self.scopedWalk = scopedWalk + self.transformRead = transformRead } func onWrite(_ hook: @escaping @Sendable (WorkspacePath) async throws -> Void) { writeHook = hook } func onRead(_ hook: @escaping @Sendable (WorkspacePath) async throws -> Void) { readHook = hook } func failNextRead() { readFailure = true } func stat(path: WorkspacePath) async throws -> FileInfo { + statCount += 1 if let statError { throw statError } return try await base.stat(path: path) } - func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + listings.append(path) + return try await base.listDirectory(path: path) + } func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { try await base.glob(pattern: pattern, currentDirectory: currentDirectory) } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } func readFile(path: WorkspacePath) async throws -> Data { reads.append(path) + if unreadable.contains(path) { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EACCES)) } let hook = readHook readHook = nil try await hook?(path) if readFailure { readFailure = false; throw TestFileSystemError.injected } - return try await base.readFile(path: path) + let data = try await base.readFile(path: path) + return transformRead?(data) ?? data } func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { writes.append(path) diff --git a/Tests/WorkspaceTests/TransactionTests.swift b/Tests/WorkspaceTests/TransactionTests.swift index 8a22bad..9004031 100644 --- a/Tests/WorkspaceTests/TransactionTests.swift +++ b/Tests/WorkspaceTests/TransactionTests.swift @@ -381,7 +381,7 @@ struct TransactionTests { } @Test(arguments: [false, true]) - func `commit can retry after capture failure or successful rollback`(duringWrite: Bool) async throws { + func `commit can retry after read failure or successful rollback`(duringWrite: Bool) async throws { let fs = HookableFileSystem() let workspace = Workspace(fileSystem: fs) try await workspace.writeText("/a", "base") @@ -435,7 +435,7 @@ struct TransactionTests { } @Test - func `tree reconstruction rejects missing or non-directory ancestry`() throws { + func `merge validation rejects missing or non-directory ancestry`() throws { let root = Workspace.MergeNode(kind: .directory, permissions: .defaultDirectory) let file = Workspace.MergeNode(kind: .file, permissions: .defaultFile, fileData: Data("keep".utf8)) let link = Workspace.MergeNode(kind: .symlink, permissions: POSIXPermissions(0o777), symlinkTarget: "elsewhere") @@ -447,7 +447,7 @@ struct TransactionTests { [.root: root, "/missing/deep": root, "/missing/deep/child": file] ] for nodes in invalid { - #expect(throws: WorkspaceError.self) { try Workspace.buildEntryTree(from: nodes) } + #expect(throws: WorkspaceError.self) { try Workspace.validateMergeNodes(nodes) } } } @@ -458,7 +458,11 @@ struct TransactionTests { let draftFS = HookableFileSystem() let draft = Workspace(fileSystem: draftFS) // Use the existing initializer to supply a draft whose I/O can suspend. - let transaction = Workspace.Transaction(parent: workspace, draft: draft, base: base, baseCheckpointID: nil, label: nil) + var nodes: [WorkspacePath: Workspace.MergeNode] = [:] + Workspace.flattenMergeNodes(base.entry, into: &nodes) + let tree = TransactionTree(nodes: nodes, load: { _ in throw WorkspaceError.readOnly }) + let transaction = Workspace.Transaction(parent: workspace, draft: draft, base: tree, + metadata: try await MetadataIndex.capture(from: workspace.fileSystem), baseCheckpointID: nil, label: nil) let entered = TestSignal() let resume = TestSignal() await draftFS.onWrite { _ in @@ -486,7 +490,11 @@ struct TransactionTests { let base = try await workspace.revisionSnapshot(.current) let draftFS = HookableFileSystem() let draft = Workspace(fileSystem: draftFS) - let transaction = Workspace.Transaction(parent: workspace, draft: draft, base: base, baseCheckpointID: nil, label: nil) + var nodes: [WorkspacePath: Workspace.MergeNode] = [:] + Workspace.flattenMergeNodes(base.entry, into: &nodes) + let tree = TransactionTree(nodes: nodes, load: { _ in throw WorkspaceError.readOnly }) + let transaction = Workspace.Transaction(parent: workspace, draft: draft, base: tree, + metadata: try await MetadataIndex.capture(from: workspace.fileSystem), baseCheckpointID: nil, label: nil) try await transaction.writeText("/a", "draft") let entered = TestSignal() let resume = TestSignal() From b7c88901b8a60210aeac6e254202f2e0272d7561 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 05:36:23 -0700 Subject: [PATCH 17/34] Capture checkpoints with the bulk traversal FileCheckpointStore's capture now walks the source with the bulk traversal when the filesystem provides one, falling back to the per-path walk otherwise. Both routes share the same reuse and persistence logic and produce byte-identical trees and pack layouts; content is read only for entries that can't be reused under the racy-mtime rule. Entries that vanish between listing and lookup are skipped on both routes instead of failing the capture. Prior-directory state is prepared lazily from cached tree entries with a binary lookup, capture-time tree-cache inserts are scan-resistant within the existing budget, and the visitor tracks ancestry with a stack instead of full-path hashing. On the medium benchmark fixture an incremental directory checkpoint drops from about 220 ms to 89 ms, and on the large fixture from 6.1 s to 1.5 s. --- .../CountingFileSystem.swift | 5 +- README.md | 10 +- Sources/Workspace/CheckpointStore.swift | 315 +++++++++++++----- .../Workspace/FS/FileSystemTraversal.swift | 21 +- Sources/Workspace/FS/LimitedFileSystem.swift | 5 +- Sources/Workspace/FS/LocalFileSystem.swift | 26 +- Sources/Workspace/FS/MountedFileSystem.swift | 11 +- Sources/Workspace/FS/SandboxFileSystem.swift | 5 +- .../FS/SecurityScopedFileSystem.swift | 5 +- Sources/Workspace/Support/CostLRU.swift | 8 +- .../CheckpointCaptureTraversalTests.swift | 305 +++++++++++++++++ .../CheckpointFormatTests.swift | 12 + .../FileSystemTraversalTests.swift | 29 ++ Tests/WorkspaceTests/TestSupport.swift | 8 +- 14 files changed, 654 insertions(+), 111 deletions(-) create mode 100644 Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift diff --git a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift index d5b8fdd..3f9beac 100644 --- a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift +++ b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift @@ -30,10 +30,11 @@ class CountingFileSystem: FileSystem, TraversingFileSystem, ExactTimestampFileSy } nonisolated package var supportsScopedWalk: Bool { base.supportsScopedWalk } - nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { call("walk") - try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories) { info, read, readLink in + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries) { info, read, readLink in self.call("walkEntry") if info.path != path { self.lock.withLock { self.counts.directoryEntries += 1 } } // The callbacks remain scoped to this awaited visit; the temporary diff --git a/README.md b/README.md index c9c8a7b..6a670fe 100644 --- a/README.md +++ b/README.md @@ -199,8 +199,10 @@ objects in `trees/`. Each object contains byte-sorted immediate child names (nev referenced by hash. An incremental capture writes only changed directories and their ancestors; unchanged objects remain shared. File reads decode only directories on the requested path. Decoded trees and the write catalog share a **32 MiB estimated decoded-storage budget** per -store (24 MiB for trees, 8 MiB for the catalog). Both use O(1) LRU eviction and refresh recency on -hits. Estimates include entry, string-buffer and bookkeeping costs; this is not a process-RSS cap. +store (24 MiB for trees, 8 MiB for the catalog). Both refresh recency on hits and normally use +O(1) LRU eviction. Capture-time tree inserts evict the most recent entry instead, so repeated +whole-tree scans retain a resident set when the trees exceed the budget. Estimates include entry, +string-buffer and bookkeeping costs; this is not a process-RSS cap. File metadata and directory listings also use path-only lookup. Full revision indexes are still materialized when a whole-tree operation requests them. Metadata capture still walks the filesystem. @@ -476,7 +478,9 @@ Authorization supports one-shot, duration-limited, and session approvals plus bo Local scans record symlinks without descending into them and retain per-entry authorization checks when using an authorized filesystem. Scans are live reads, not atomic snapshots: concurrent changes can affect the result or cause an error. Snapshot capture fails when a -listed child disappears before it can be inspected or read. Scoped no-follow opens reject +listed child disappears before it can be inspected or read. Directory-backed checkpoints omit +listed children that vanish before their metadata lookup (ENOENT/ENOTDIR); failures after metadata, +including content reads, still abort the checkpoint. Scoped no-follow opens reject substituted symlinks even when they point inside the root. Deep scans bound open directory handles by falling back to ordinary per-path operations; no handles are cached between calls. As with other local operations, host processes can relocate open directories or mutate diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 1b60dfc..575584e 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -508,7 +508,7 @@ actor FileCheckpointStore: CheckpointStore { } /// Names are validated before entering the cache; no absolute paths are persisted. - private struct TreeEntry: Codable, Sendable { + private struct TreeEntry: Codable, Sendable, Equatable { var name: String var kind: SnapshotManifest.Node.Kind var permissions: POSIXPermissions? @@ -519,9 +519,15 @@ actor FileCheckpointStore: CheckpointStore { var treeHash: String? var packed: PackedBlob? - init(_ node: SnapshotManifest.Node) throws { - name = node.path.name - try Self.validate(name) + init(_ node: SnapshotManifest.Node, name: String? = nil) throws { + self.init(metadataOf: node, name: name ?? node.path.name) + try Self.validate(self.name) + } + + /// Prior leaf metadata can also represent a file at the snapshot root. + /// This initializer is never used to publish or cache directory entries. + init(metadataOf node: SnapshotManifest.Node, name: String) { + self.name = name kind = node.kind permissions = node.permissions contentHash = node.contentHash @@ -540,11 +546,34 @@ actor FileCheckpointStore: CheckpointStore { } func node(in parent: WorkspacePath) -> SnapshotManifest.Node { - .init(kind: kind, path: parent.appending(name), permissions: permissions, + node(at: parent.appending(name)) + } + + func node(at path: WorkspacePath) -> SnapshotManifest.Node { + .init(kind: kind, path: path, permissions: permissions, contentHash: contentHash, contentSize: contentSize, modificationDate: modificationDate, symlinkTarget: symlinkTarget, treeHash: treeHash, packed: packed) } + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.name.utf8.elementsEqual(rhs.name.utf8) && lhs.kind == rhs.kind && + lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && + lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && + lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && + lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed + } + + /// Trees are validated and sorted by exact name bytes before being cached. + static func index(of name: String, in entries: [Self]) -> Int? { + var low = 0, high = entries.count + while low < high { + let mid = (low + high) / 2 + if entries[mid].name.utf8.lexicographicallyPrecedes(name.utf8) { low = mid + 1 } + else { high = mid } + } + return low < entries.count && entries[low].name.utf8.elementsEqual(name.utf8) ? low : nil + } + var estimatedBytes: Int { // String buffers, array storage, optional fields and allocator overhead. var stringBytes = name.utf8.count @@ -580,9 +609,8 @@ actor FileCheckpointStore: CheckpointStore { batch.summarizeInsertions = parentManifest == nil let captureStartedAt = Date() let snapshotID = UUID() - let root = try await self.captureManifestNode( + let root = try await self.captureManifest( from: filesystem, - at: .root, reusing: parentManifest?.root, previousCaptureStartedAt: parentManifest?.captureStartedAt, workspaceId: draft.workspaceID, @@ -594,9 +622,9 @@ actor FileCheckpointStore: CheckpointStore { ) let summary: ChangeSet.Summary if let parentManifest { - summary = try await self.compareNodes(parentManifest.root, root, workspaceId: draft.workspaceID).summary + summary = try await self.compareNodes(parentManifest.root, root, workspaceId: draft.workspaceID, capturing: true).summary } else { - summary = try self.creationSummary(root, workspaceId: draft.workspaceID, insertedLines: batch.insertedLines) + summary = try self.creationSummary(root, workspaceId: draft.workspaceID, insertedLines: batch.insertedLines, capturing: true) } let checkpoint = Checkpoint.make( snapshotID: snapshotID, @@ -641,6 +669,142 @@ actor FileCheckpointStore: CheckpointStore { return manifest } + /// A frame keeps cached name-only entries intact. Only a requested old child + /// becomes a Node; legacy recursive manifests retain their embedded children. + private struct CaptureDirectory { + var path: WorkspacePath + var name: String + var childOffset: Int + var previous: SnapshotManifest.Node? + var previousEntries: [TreeEntry] + var legacyChildren: [SnapshotManifest.Node]? + var children: [TreeEntry] = [] + + func previousEntry(named name: String) -> TreeEntry? { + guard let index = TreeEntry.index(of: name, in: previousEntries) else { return nil } + return previousEntries[index] + } + + func previousNode(named name: String, at path: WorkspacePath) -> SnapshotManifest.Node? { + guard let index = TreeEntry.index(of: name, in: previousEntries) else { return nil } + return legacyChildren?[index] ?? previousEntries[index].node(at: path) + } + } + + private func captureManifest( + from filesystem: any FileSystem, + reusing previous: SnapshotManifest.Node?, + previousCaptureStartedAt: Date?, + workspaceId: UUID, + batch: BlobBatch + ) async throws -> SnapshotManifest.Node { + guard filesystem.supportsScopedWalk else { + return try await captureManifestNode(from: filesystem, at: .root, reusing: previous, + previousCaptureStartedAt: previousCaptureStartedAt, workspaceId: workspaceId, batch: batch) + } + var stack: [CaptureDirectory] = [] + var root = SnapshotManifest.Node(kind: .missing, path: .root) + try await filesystem.walk(skippingVanishedEntries: true) { info, read, readLink in + try Task.checkCancellation() + let path = info.path + let node: SnapshotManifest.Node + let name: String + if info.kind == .directory, stack.last?.path == path { + // A nonempty directory closes after its descendants. Its name and + // parent are already known; there is no path lookup on this branch. + let directory = stack.removeLast() + name = directory.name + node = try self.finishCaptureDirectory(info, directory: directory, workspaceId: workspaceId) + } else if path.isRoot { + name = "/" + if info.kind == .directory { + node = try self.finishCaptureDirectory(info, + directory: self.captureDirectory(reusing: previous, at: path, name: name, workspaceId: workspaceId), + workspaceId: workspaceId) + } else { + node = try await self.captureLeaf(info, reusing: previous.map { TreeEntry(metadataOf: $0, name: "/") }, + previousCaptureStartedAt: previousCaptureStartedAt, workspaceId: workspaceId, + batch: batch, read: read, readLink: readLink) + } + } else { + if stack.isEmpty { + stack.append(try self.captureDirectory(reusing: previous, at: .root, name: "/", workspaceId: workspaceId)) + } + // Postorder first reaches a new directory through a descendant. + // Parse only the suffix below the current frame; ordinary siblings + // need just their stored name, without hashing/constructing parents. + guard path != stack.last!.path, path.isWithin(stack.last!.path) else { + // A walker that breaks depth-first postorder would otherwise build wrong trees. + throw WorkspaceError.storageCorrupted("capture walk visited \(path) outside \(stack.last!.path)") + } + var suffix = path.string.utf8.dropFirst(stack.last!.childOffset) + while let slash = suffix.firstIndex(of: 47) { + let component = String(decoding: suffix[.. CaptureDirectory { + let entries: [TreeEntry] + let legacy: [SnapshotManifest.Node]? + if let previous, previous.kind == .directory { + if let hash = previous.treeHash { + entries = try treeEntries(hash: hash, workspaceId: workspaceId, capturing: true) + legacy = nil + } else { + legacy = (previous.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) } + entries = try legacy!.map { try TreeEntry($0) } + for index in entries.indices.dropFirst() { + guard PathName(entries[index - 1].name) < PathName(entries[index].name) else { + throw WorkspaceError.storageCorrupted("snapshot directory \(path) contains duplicate child \(entries[index].name)") + } + } + } + } else { entries = []; legacy = nil } + return CaptureDirectory(path: path, name: name, childOffset: path.isRoot ? 1 : path.string.utf8.count + 1, + previous: previous, previousEntries: entries, legacyChildren: legacy) + } + + private func finishCaptureDirectory( + _ info: FileInfo, directory: CaptureDirectory, workspaceId: UUID + ) throws -> SnapshotManifest.Node { + let entries = directory.children.sorted { PathName($0.name) < PathName($1.name) } + if let previous = directory.previous, previous.kind == .directory, previous.treeHash != nil, + directory.previousEntries == entries { + var reused = previous + reused.permissions = info.permissions + reused.modificationDate = info.modificationDate + return reused + } + return try persistDirectory(SnapshotManifest.Node( + kind: .directory, path: info.path, permissions: info.permissions, modificationDate: info.modificationDate + ), entries: entries, workspaceId: workspaceId, capturing: true) + } + private func captureManifestNode( from filesystem: any FileSystem, at path: WorkspacePath, @@ -650,11 +814,39 @@ actor FileCheckpointStore: CheckpointStore { batch: BlobBatch ) async throws -> SnapshotManifest.Node { try Task.checkCancellation() - // exists follows the final symlink on some backends, losing dangling - // links and cycles. Capture the entry itself, just as Snapshot.capture does. guard let info = try await Snapshot.infoIfPresent(at: path, on: filesystem) else { return SnapshotManifest.Node(kind: .missing, path: path) } + if info.kind == .directory { + var directory = try captureDirectory(reusing: previous, at: path, name: path.name, workspaceId: workspaceId) + for entry in try await filesystem.listDirectory(path: path).sorted(by: { PathName(walkSortKey($0)) < PathName(walkSortKey($1)) }) { + try TreeEntry.validate(entry.name) + let childPath = path.appending(entry.name) + let child = try await captureManifestNode(from: filesystem, at: childPath, + reusing: directory.previousNode(named: entry.name, at: childPath), + previousCaptureStartedAt: previousCaptureStartedAt, workspaceId: workspaceId, batch: batch) + // A listed child can vanish before its own lstat. Skip it on both + // routes, while content read failures after metadata still propagate. + if child.kind != .missing { directory.children.append(try TreeEntry(child, name: entry.name)) } + } + return try finishCaptureDirectory(info, directory: directory, workspaceId: workspaceId) + } + return try await captureLeaf(info, reusing: previous.map { TreeEntry(metadataOf: $0, name: path.name) }, previousCaptureStartedAt: previousCaptureStartedAt, + workspaceId: workspaceId, batch: batch, + read: { _ in try await filesystem.readFile(path: path) }, + readLink: { try await filesystem.readSymlink(path: path) }) + } + + private func captureLeaf( + _ info: FileInfo, + reusing previous: TreeEntry?, + previousCaptureStartedAt: Date?, + workspaceId: UUID, + batch: BlobBatch, + read: @Sendable (Int?) async throws -> Data, + readLink: @Sendable () async throws -> String + ) async throws -> SnapshotManifest.Node { + let path = info.path switch info.kind { case .file: // Conservatively cover second-resolution and two-second-resolution timestamps. @@ -670,7 +862,7 @@ actor FileCheckpointStore: CheckpointStore { let hash = previous?.contentHash, Self.isValidContentHash(hash), let previous, - blobExists(previous, workspaceId: workspaceId, batch: batch) { + blobExists(previous.node(at: path), workspaceId: workspaceId, batch: batch) { return SnapshotManifest.Node( kind: .file, path: path, @@ -681,7 +873,7 @@ actor FileCheckpointStore: CheckpointStore { packed: previous.packed ) } - let data = try await filesystem.readFile(path: path) + let data = try await read(nil) let hash = SHA256.hexDigest(of: data) if batch.summarizeInsertions, data.count <= 1_000_000, !data.withUnsafeBytes({ $0.contains(0) }), let text = String(data: data, encoding: .utf8) { @@ -697,52 +889,16 @@ actor FileCheckpointStore: CheckpointStore { modificationDate: info.modificationDate, packed: packed ) - case .directory: - var previousChildren: [PathName: SnapshotManifest.Node] = [:] - let previousEntries = try previous.map { try self.children(of: $0, workspaceId: workspaceId) } ?? [] - for child in previousEntries { - guard previousChildren.updateValue(child, forKey: PathName(child.path.name)) == nil else { - throw WorkspaceError.storageCorrupted( - "snapshot directory \(path) contains duplicate child \(child.path.name)" - ) - } - } - var children: [SnapshotManifest.Node] = [] - for entry in try await filesystem.listDirectory(path: path).sorted(by: { PathName($0.name) < PathName($1.name) }) { - try TreeEntry.validate(entry.name) - children.append( - try await captureManifestNode( - from: filesystem, - at: path.appending(entry.name), - reusing: previousChildren[PathName(entry.name)], - previousCaptureStartedAt: previousCaptureStartedAt, - workspaceId: workspaceId, - batch: batch - ) - ) - } - if let previous, previous.kind == .directory, previous.treeHash != nil, - previousEntries == children { - var reused = previous - reused.permissions = info.permissions - reused.modificationDate = info.modificationDate - return reused - } - return try persistDirectory(SnapshotManifest.Node( - kind: .directory, - path: path, - permissions: info.permissions, - modificationDate: info.modificationDate, - children: children - ), workspaceId: workspaceId) case .symlink: return SnapshotManifest.Node( kind: .symlink, path: path, permissions: info.permissions, modificationDate: info.modificationDate, - symlinkTarget: try await filesystem.readSymlink(path: path) + symlinkTarget: try await readLink() ) + case .directory: + preconditionFailure("directories must be captured postorder") } } @@ -790,7 +946,8 @@ actor FileCheckpointStore: CheckpointStore { private func revisionEntry( from node: SnapshotManifest.Node, workspaceId: UUID, - depth: Int? = nil + depth: Int? = nil, + capturing: Bool = false ) throws -> RevisionIndex.Entry { switch node.kind { case .missing: @@ -809,8 +966,8 @@ actor FileCheckpointStore: CheckpointStore { return .directory( path: node.path, permissions: node.permissions ?? .defaultDirectory, - children: depth == 0 ? [] : try children(of: node, workspaceId: workspaceId).map { - try revisionEntry(from: $0, workspaceId: workspaceId, depth: depth.map { $0 - 1 }) + children: depth == 0 ? [] : try children(of: node, workspaceId: workspaceId, capturing: capturing).map { + try revisionEntry(from: $0, workspaceId: workspaceId, depth: depth.map { $0 - 1 }, capturing: capturing) } ) case .symlink: @@ -832,13 +989,8 @@ actor FileCheckpointStore: CheckpointStore { guard node.kind == .directory else { return nil } if let hash = node.treeHash { let entries = try treeEntries(hash: hash, workspaceId: workspaceId) - var low = 0, high = entries.count - while low < high { - let mid = (low + high) / 2 - if PathName(entries[mid].name) < name { low = mid + 1 } else { high = mid } - } - guard low < entries.count, PathName(entries[low].name) == name else { return nil } - node = entries[low].node(in: node.path) + guard let index = TreeEntry.index(of: component, in: entries) else { return nil } + node = entries[index].node(in: node.path) } else { // Legacy manifests already contain absolute child nodes. guard let child = node.children?.first(where: { PathName($0.path.name) == name }) else { return nil } @@ -1960,19 +2112,19 @@ actor FileCheckpointStore: CheckpointStore { try Self.syncDirectory(treesDirectoryURL(workspaceId: workspaceId)) } - private func cacheTree(_ entries: [TreeEntry], hash: String, workspaceId: UUID) { + private func cacheTree(_ entries: [TreeEntry], hash: String, workspaceId: UUID, capturing: Bool = false) { let key = workspaceId.uuidString + hash let cost = 256 + key.utf8.count * 2 + entries.reduce(0) { $0 + $1.estimatedBytes } - treeCache.insert(entries, for: key, cost: cost) + treeCache.insert(entries, for: key, cost: cost, evicting: capturing ? .mostRecent : .leastRecent) } - private func children(of node: SnapshotManifest.Node, workspaceId: UUID) throws -> [SnapshotManifest.Node] { + private func children(of node: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) throws -> [SnapshotManifest.Node] { guard node.kind == .directory else { return [] } guard let hash = node.treeHash else { return (node.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) } } - return try treeEntries(hash: hash, workspaceId: workspaceId).map { $0.node(in: node.path) } + return try treeEntries(hash: hash, workspaceId: workspaceId, capturing: capturing).map { $0.node(in: node.path) } } - private func treeEntries(hash: String, workspaceId: UUID) throws -> [TreeEntry] { + private func treeEntries(hash: String, workspaceId: UUID, capturing: Bool = false) throws -> [TreeEntry] { guard Self.isValidContentHash(hash) else { throw WorkspaceError.storageCorrupted("invalid tree hash") } let url = treesDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash) guard fileManager.fileExists(atPath: url.path) else { throw WorkspaceError.storageCorrupted("missing tree \(hash)") } @@ -1991,7 +2143,7 @@ actor FileCheckpointStore: CheckpointStore { } previous = PathName(entry.name) } - cacheTree(entries, hash: hash, workspaceId: workspaceId) + cacheTree(entries, hash: hash, workspaceId: workspaceId, capturing: capturing) } return entries } @@ -2001,14 +2153,21 @@ actor FileCheckpointStore: CheckpointStore { ) throws -> SnapshotManifest.Node { let entries = try (node.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) - }.map(TreeEntry.init) + }.map { try TreeEntry($0) } + return try persistDirectory(node, entries: entries, workspaceId: workspaceId, syncParent: syncParent) + } + + private func persistDirectory( + _ node: SnapshotManifest.Node, entries: [TreeEntry], workspaceId: UUID, + syncParent: Bool = false, capturing: Bool = false + ) throws -> SnapshotManifest.Node { let bytes = try compactEncoder.encode(entries) let hash = SHA256.hexDigest(of: bytes) let url = treesDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash) if !fileManager.fileExists(atPath: url.path) { try Self.durableWrite(bytes, to: url, syncParent: syncParent) } - cacheTree(entries, hash: hash, workspaceId: workspaceId) + cacheTree(entries, hash: hash, workspaceId: workspaceId, capturing: capturing) var reference = node reference.children = nil reference.treeHash = hash @@ -2142,14 +2301,14 @@ actor FileCheckpointStore: CheckpointStore { } } - private func creationSummary(_ root: SnapshotManifest.Node, workspaceId: UUID, insertedLines: [WorkspacePath: Int]) throws -> ChangeSet.Summary { + private func creationSummary(_ root: SnapshotManifest.Node, workspaceId: UUID, insertedLines: [WorkspacePath: Int], capturing: Bool = false) throws -> ChangeSet.Summary { var paths: [WorkspacePath] = [] var files = 0 func visit(_ node: SnapshotManifest.Node) throws { if node.kind == .missing { return } paths.append(node.path) if node.kind == .file { files += 1 } - for child in try children(of: node, workspaceId: workspaceId) { try visit(child) } + for child in try children(of: node, workspaceId: workspaceId, capturing: capturing) { try visit(child) } } try visit(root) let additions = insertedLines.values.reduce(0, +) @@ -2157,19 +2316,19 @@ actor FileCheckpointStore: CheckpointStore { statistics: .init(changedPathCount: paths.count, changedFileCount: files, additions: additions, deletions: 0)) } - private func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID) async throws -> ChangeSet { + private func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) async throws -> ChangeSet { if old == new { return ChangeSet(changes: []) } if old.kind == .directory, new.kind == .directory { var changes: [FileChange] = (old.permissions ?? .defaultDirectory) == (new.permissions ?? .defaultDirectory) ? [] : [.init(path: new.path, kind: .directory, effect: .modified)] var omissions: [TextDiffOmission] = [] if old.treeHash != nil, old.treeHash == new.treeHash { return ChangeSet(changes: changes) } - let before = Dictionary(uniqueKeysWithValues: try children(of: old, workspaceId: workspaceId).map { (PathName($0.path.name), $0) }) - let after = Dictionary(uniqueKeysWithValues: try children(of: new, workspaceId: workspaceId).map { (PathName($0.path.name), $0) }) + let before = Dictionary(uniqueKeysWithValues: try children(of: old, workspaceId: workspaceId, capturing: capturing).map { (PathName($0.path.name), $0) }) + let after = Dictionary(uniqueKeysWithValues: try children(of: new, workspaceId: workspaceId, capturing: capturing).map { (PathName($0.path.name), $0) }) for key in Set(before.keys).union(after.keys).sorted() { let name = key.string let result = try await compareNodes( before[key] ?? .init(kind: .missing, path: old.path.appending(name)), - after[key] ?? .init(kind: .missing, path: new.path.appending(name)), workspaceId: workspaceId) + after[key] ?? .init(kind: .missing, path: new.path.appending(name)), workspaceId: workspaceId, capturing: capturing) changes += result.changes omissions += result.textDiffOmissions } @@ -2177,8 +2336,8 @@ actor FileCheckpointStore: CheckpointStore { } // Newly created/deleted trees need their complete summaries; normal incremental // changes reach this path only for changed leaves. - let before = RevisionIndex(id: UUID(), root: old.path, entry: try revisionEntry(from: old, workspaceId: workspaceId)) - let after = RevisionIndex(id: UUID(), root: new.path, entry: try revisionEntry(from: new, workspaceId: workspaceId)) + let before = RevisionIndex(id: UUID(), root: old.path, entry: try revisionEntry(from: old, workspaceId: workspaceId, capturing: capturing)) + let after = RevisionIndex(id: UUID(), root: new.path, entry: try revisionEntry(from: new, workspaceId: workspaceId, capturing: capturing)) return try await ChangeSet.compare(before: before, after: after, maxTextBytes: 1_000_000, loadBefore: { path in try await self.readNodeFile(path, root: old, workspaceId: workspaceId) }, loadAfter: { path in try await self.readNodeFile(path, root: new, workspaceId: workspaceId) }) diff --git a/Sources/Workspace/FS/FileSystemTraversal.swift b/Sources/Workspace/FS/FileSystemTraversal.swift index 33e9893..409dd85 100644 --- a/Sources/Workspace/FS/FileSystemTraversal.swift +++ b/Sources/Workspace/FS/FileSystemTraversal.swift @@ -9,8 +9,10 @@ import Glibc /// never descended into. A missing root produces no visits. Read callbacks are /// nonescaping and valid only during their visit. This is a live read-only walk, /// not an atomic snapshot: errors from children disappearing during enumeration or -/// reads propagate, except directory errors explicitly allowed by the caller's skip -/// option. Local scoped no-follow opens reject substitutions with symlinks, even +/// reads propagate. The caller can skip children missing at their metadata lookup +/// (ENOENT/ENOTDIR), independently of skipping unreadable directories. Read errors +/// after metadata always propagate. Local scoped no-follow opens reject substitutions +/// with symlinks, even /// when the replacement link points inside the root. Very deep local subtrees fall /// back to ordinary per-path operations with a root identity check at the handoff. /// @@ -19,7 +21,7 @@ import Glibc package protocol TraversingFileSystem: FileSystem { /// Forwarders return false when their base needs the per-path fallback. var supportsScopedWalk: Bool { get } - nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool, visit: FileSystemWalkVisitor) async throws } @@ -38,20 +40,24 @@ extension FileSystem { (self as? any TraversingFileSystem)?.supportsScopedWalk ?? false } nonisolated(nonsending) package func walk(at path: WorkspacePath = .root, skippingUnreadableDirectories: Bool = false, + skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { if let traversal = self as? any TraversingFileSystem { - try await traversal.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await traversal.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } else { - try await walkUsingPaths(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await walkUsingPaths(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } } // In particular, never unwrap AuthorizedFileSystem here: each stat, listing and // requested content read must pass through its normal per-entry checks. nonisolated(nonsending) package func walkUsingPaths(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + skippingVanishedEntries: Bool = false, requireExisting: Bool = false, visit: FileSystemWalkVisitor) async throws { let info: FileInfo - if requireExisting { info = try await stat(path: path) } + if requireExisting && !skippingVanishedEntries { info = try await stat(path: path) } else { guard let existing = try await Snapshot.infoIfPresent(at: path, on: self) else { return } info = existing @@ -69,7 +75,8 @@ extension FileSystem { $0.1.utf8.lexicographicallyPrecedes($1.1.utf8) }) { try await walkUsingPaths(at: path.appending(child.name), - skippingUnreadableDirectories: skippingUnreadableDirectories, requireExisting: true, visit: visit) + skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, requireExisting: true, visit: visit) } } try await visit(info, { length in diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index 8b46fcd..f1a7e70 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -61,9 +61,10 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti nonisolated package var supportsScopedWalk: Bool { base.supportsScopedWalk } - nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { - try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } public func stat(path: WorkspacePath) async throws -> FileInfo { try await base.stat(path: path) } diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index f8e6078..0df18da 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -556,7 +556,7 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem /// Holds only the active directory ancestry, for this read-only call. Descents and /// file opens use O_NOFOLLOW; metadata and link targets use the same parent fd. - nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { let entry: Entry let info: LocalStat @@ -568,20 +568,28 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem if Snapshot.isMissingEntry(error) { return } throw error } - try await walkEntry(entry, info: info, depth: 0, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await walkEntry(entry, info: info, depth: 0, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } nonisolated(nonsending) private func walkEntry(_ entry: Entry, info: LocalStat, depth: Int, skippingUnreadableDirectories: Bool, + skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { try Task.checkCancellation() // Bound this call's retained ancestry, leaving room for unrelated operations at // the usual 256-descriptor soft limit. The fallback resolves each operation anew. if depth == 32 { - let current = try self.metadata(resolve(entry.path, followFinal: false)) + let current: LocalStat + do { current = try self.metadata(resolve(entry.path, followFinal: false)) } + catch { + guard skippingVanishedEntries, Snapshot.isMissingEntry(error) else { throw error } + return + } guard current.st_dev == info.st_dev, current.st_ino == info.st_ino else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ESTALE)) } try await walkUsingPaths(at: entry.path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, requireExisting: true, visit: visit) return } @@ -591,9 +599,14 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem let children: [(entry: Entry, info: LocalStat, key: String)] do { directory = try openEntry(entry, flags: O_RDONLY | O_DIRECTORY) - children = try names(in: directory, consumingOffset: true).map { name in + children = try names(in: directory, consumingOffset: true).compactMap { name -> (entry: Entry, info: LocalStat, key: String)? in let child = try Entry(parent: directory, storedName: name, path: entry.path.appending(name)) - let info = try self.metadata(child) + let info: LocalStat + do { info = try self.metadata(child) } + catch { + guard skippingVanishedEntries, Snapshot.isMissingEntry(error) else { throw error } + return nil + } return (child, info, walkSortKey(name: name, isDirectory: info.st_mode & S_IFMT == S_IFDIR)) }.sorted { $0.key.utf8.lexicographicallyPrecedes($1.key.utf8) } } catch { @@ -603,7 +616,8 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem return } for (child, info, _) in children { - try await walkEntry(child, info: info, depth: depth + 1, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await walkEntry(child, info: info, depth: depth + 1, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } } try await visit(metadata, { length in try self.readEntry(entry, length: length) }, diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index 7165535..b16562f 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -81,7 +81,7 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS base.supportsScopedWalk || mountsSnapshot().contains { $0.fileSystem.supportsScopedWalk } } - nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { let mounts = mountsSnapshot() // An entire subtree in one backend can use its descriptor-scoped walk. @@ -91,7 +91,8 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS var visited = false do { try await resolved.filesystem.walk(at: resolved.relativePath, - skippingUnreadableDirectories: skippingUnreadableDirectories) { info, read, readLink in + skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries) { info, read, readLink in visited = true let virtual = resolved.mountPoint.appending(info.path.components.joined(separator: "/")) try await withoutActuallyEscaping(read) { scopedRead in @@ -105,7 +106,8 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS // A backend root may be a skippable child in our virtual namespace. guard !visited, resolved.relativePath.isRoot, skippingUnreadableDirectories, isSkippableWalkError(error, at: path) else { throw error } - try await walkUsingPaths(at: path, skippingUnreadableDirectories: true, visit: visit) + try await walkUsingPaths(at: path, skippingUnreadableDirectories: true, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } return } @@ -120,7 +122,8 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS return } for child in listing.sorted(by: { PathName(walkSortKey($0)) < PathName(walkSortKey($1)) }) { - try await walk(at: child.info.path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await walk(at: child.info.path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } } try await visit(info, { length in diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index 101a7bc..a87fb04 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -25,9 +25,10 @@ public final class SandboxFileSystem: FileSystem, TraversingFileSystem, PreviewS try backing.applyConfiguration(rootDirectory: resolvedRoot) } - nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { - try await backing.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await backing.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } /// See ``FileSystem/stat(path:)``. diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index 0f1feb4..6b87164 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -128,9 +128,10 @@ public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, P } } - nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { - try await backing.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await backing.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } /// See ``FileSystem/stat(path:)``. diff --git a/Sources/Workspace/Support/CostLRU.swift b/Sources/Workspace/Support/CostLRU.swift index f81cf51..f927f29 100644 --- a/Sources/Workspace/Support/CostLRU.swift +++ b/Sources/Workspace/Support/CostLRU.swift @@ -1,6 +1,8 @@ /// A byte-budgeted LRU. Dictionary links make lookup, promotion and eviction O(1). /// Costs include estimated decoded storage and bookkeeping, not encoded JSON sizes. struct CostLRU: Sendable { + enum Eviction { case leastRecent, mostRecent } + private struct Entry: Sendable { var value: Value var cost: Int @@ -23,10 +25,12 @@ struct CostLRU: Sendable { return entry.value } - mutating func insert(_ value: Value, for key: Key, cost entryCost: Int) { + /// MRU eviction lets a whole-tree scan retain most of the resident set. + /// Lookup promotion and the decoded-storage bound are the same for both policies. + mutating func insert(_ value: Value, for key: Key, cost entryCost: Int, evicting eviction: Eviction = .leastRecent) { remove(key) guard entryCost >= 0, entryCost <= limit else { return } - while cost > limit - entryCost, let key = oldest { remove(key) } + while cost > limit - entryCost, let key = (eviction == .mostRecent ? newest : oldest) { remove(key) } entries[key] = Entry(value: value, cost: entryCost, previous: newest) if let newest { entries[newest]?.next = key } else { oldest = key } newest = key diff --git a/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift b/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift new file mode 100644 index 0000000..b947465 --- /dev/null +++ b/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift @@ -0,0 +1,305 @@ +import Foundation +import Testing +@testable import Workspace +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +struct CheckpointCaptureTraversalTests { + @Test(arguments: [Workspace.CheckpointCapturePolicy.trustMetadata, .verifyContents]) + func captureRoutesPersistIdenticalTrees(policy: Workspace.CheckpointCapturePolicy) async throws { + let container = try TestSupport.temporaryDirectory("CaptureRoutes") + defer { TestSupport.remove(container) } + let files = container.appendingPathComponent("files") + let local = try LocalFileSystem(root: files) + try await local.createDirectory(path: "/a/empty", recursive: true) + try await local.createFile(path: "/a/file", data: Data([0, 1, 255])) + try await local.createFile(path: "/a-", data: Data("ordering\n".utf8)) + try await local.setPermissions(path: "/a/file", permissions: POSIXPermissions(0o640)) + try await local.setPermissions(path: "/a", permissions: POSIXPermissions(0o750)) + try await local.createHardLink(path: "/hard", target: "/a/file") + for (name, target) in [("dangling", "missing"), ("cycle", "cycle"), + ("alias", "a"), ("escape", container.appendingPathComponent("outside").path)] { + try FileManager.default.createSymbolicLink(atPath: files.appendingPathComponent(name).path, + withDestinationPath: target) + } + try Data("outside".utf8).write(to: container.appendingPathComponent("outside")) + // macOS volumes cannot host both spellings. A real mounted backend can, + // and also exercises forwarding plus mixed bulk/per-path traversal. + let unicode = InMemoryFileSystem() + try await unicode.createFile(path: "/caf\u{e9}", data: Data([1])) + try await unicode.createFile(path: "/cafe\u{301}", data: Data([2])) + let source = MountedFileSystem(base: local, mounts: [.init(mountPoint: "/unicode", fileSystem: unicode)]) + let bulk = CaptureSource(source, scoped: true) + let paths = CaptureSource(source, scoped: false) + let id = UUID() + let roots = [container.appendingPathComponent("bulk"), container.appendingPathComponent("paths")] + let stores = roots.map { FileCheckpointStore(rootDirectory: $0, checkpointCapture: policy) } + let old = Date(timeIntervalSince1970: 1) + for name in ["a/file", "a-"] { + try FileManager.default.setAttributes([.modificationDate: old], ofItemAtPath: files.appendingPathComponent(name).path) + } + var parents: [UUID?] = [nil, nil] + var priorTrees: [String: Data] = [:] + for phase in 0..<4 { + if phase == 2 { + try await local.createFile(path: "/a/new", data: Data("new\n".utf8)) + try await local.setPermissions(path: "/a/file", permissions: POSIXPermissions(0o600)) + try await unicode.writeFile(path: "/cafe\u{301}", data: Data([3]), append: false) + } + if phase == 3 { + try await local.remove(path: "/a/new", recursive: false) + try await local.remove(path: "/alias", recursive: false) + try await local.createSymlink(path: "/alias", target: "missing") + } + bulk.reset() + paths.reset() + var manifests: [Data] = [] + var summaries: [ChangeSet.Summary] = [] + for (i, fs) in [bulk, paths].enumerated() { + let checkpoint = try await stores[i].captureRevision(from: fs, draft: .init( + workspaceID: id, label: nil, preferredParentID: parents[i], origin: .manual)) + parents[i] = checkpoint.id + summaries.append(checkpoint.summary) + let url = roots[i].appendingPathComponent(id.uuidString) + .appendingPathComponent("snapshots/\(checkpoint.snapshotId).json") + var manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: url)) as? [String: Any]) + // These are intentionally fresh for every capture; all persisted node + // metadata, hashes, targets and ordering must match byte for byte. + manifest.removeValue(forKey: "id") + manifest.removeValue(forKey: "captureStartedAt") + manifests.append(try JSONSerialization.data(withJSONObject: manifest, options: [.sortedKeys])) + } + #expect(manifests[0] == manifests[1]) + #expect(summaries[0] == summaries[1]) + let trees = try artifacts(roots[0].appendingPathComponent(id.uuidString).appendingPathComponent("trees")) + #expect(try trees == artifacts(roots[1].appendingPathComponent(id.uuidString).appendingPathComponent("trees"))) + if phase == 1 { #expect(trees == priorTrees) } + priorTrees = trees + #expect(bulk.walkCount == 1) + #expect(bulk.statCount == 0 && bulk.listCount == 0) + #expect(bulk.reads == paths.reads) + if phase == 1, policy == .trustMetadata { + #expect(!bulk.reads.contains("/a/file")) + #expect(!bulk.reads.contains("/hard")) + #expect(!bulk.reads.contains("/a-")) + } + if policy == .verifyContents { #expect(bulk.reads.contains("/a/file")) } + } + } + + @Test(arguments: [false, true]) + func racyTimestampsAndMissingBlobsForceReads(scoped: Bool) async throws { + let container = try TestSupport.temporaryDirectory("CaptureReuse") + defer { TestSupport.remove(container) } + let files = container.appendingPathComponent("files") + let local = try LocalFileSystem(root: files) + let source = CaptureSource(local, scoped: scoped) + let dates = ["stable": Date(timeIntervalSince1970: 1), "racy": Date().addingTimeInterval(60)] + for (name, date) in dates { + try await local.createFile(path: WorkspacePath(normalizing: "/" + name), data: Data("old".utf8)) + try FileManager.default.setAttributes([.modificationDate: date], ofItemAtPath: files.appendingPathComponent(name).path) + } + let id = UUID() + let root = container.appendingPathComponent("store") + let store = FileCheckpointStore(rootDirectory: root) + let first = try await store.captureRevision(from: source, draft: .init( + workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + for (name, date) in dates { + try await local.writeFile(path: WorkspacePath(normalizing: "/" + name), data: Data("new".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: date], ofItemAtPath: files.appendingPathComponent(name).path) + } + source.reset() + let second = try await store.captureRevision(from: source, draft: .init( + workspaceID: id, label: nil, preferredParentID: first.id, origin: .manual)) + #expect(source.reads == ["/racy"]) + #expect(second.summary.touchedPaths == ["/racy"]) + #expect(try await store.readSnapshotFile(id: second.snapshotId, workspaceId: id, path: "/racy", offset: 0, length: nil) == Data("new".utf8)) + // Repair an absent CAS object with the same bytes. A different payload + // would still fail when the summary tries to read the damaged parent. + try await local.writeFile(path: "/stable", data: Data("old".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: dates["stable"]!], ofItemAtPath: files.appendingPathComponent("stable").path) + let hash = SHA256.hexDigest(of: Data("old".utf8)) + try FileManager.default.removeItem(at: root.appendingPathComponent(id.uuidString).appendingPathComponent("blobs/" + hash)) + source.reset() + let repaired = try await store.captureRevision(from: source, draft: .init( + workspaceID: id, label: nil, preferredParentID: second.id, origin: .manual)) + #expect(source.reads == ["/racy", "/stable"]) + #expect(try await store.readSnapshotFile(id: repaired.snapshotId, workspaceId: id, path: "/stable", offset: 0, length: nil) == Data("old".utf8)) + } + + @Test(arguments: [Int32(ENOENT), Int32(ENOTDIR)]) + func vanishedListedChildrenAreOmittedByBothRoutes(code: Int32) async throws { + let root = try TestSupport.temporaryDirectory("CaptureVanished") + defer { TestSupport.remove(root) } + let base = InMemoryFileSystem() + try await base.createFile(path: "/real", data: Data([1])) + let ghost = GhostFileSystem(base: base, code: code) + do { + try await ghost.walk { _, _, _ in } + Issue.record("ordinary walks must still report vanished children") + } catch { #expect((error as NSError).code == Int(code)) } + let id = UUID() + var trees: [[String: Data]] = [] + for scoped in [false, true] { + let directory = root.appendingPathComponent(scoped ? "walk" : "paths") + let store = FileCheckpointStore(rootDirectory: directory) + let checkpoint = try await store.captureRevision(from: CaptureSource(ghost, scoped: scoped), draft: .init( + workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + let snapshot = try #require(await store.loadSnapshot(id: checkpoint.snapshotId, workspaceId: id)) + guard case let .directory(captured) = snapshot.entry else { Issue.record("expected directory"); return } + #expect(captured.children.map(\.path) == ["/real"]) + #expect(checkpoint.summary.touchedPaths == [.root, "/real"]) + trees.append(try artifacts(directory.appendingPathComponent(id.uuidString).appendingPathComponent("trees"))) + } + #expect(trees[0] == trees[1]) + } + + @Test + func captureRoutesProduceIdenticalPackLayout() async throws { + let container = try TestSupport.temporaryDirectory("CapturePacks") + defer { TestSupport.remove(container) } + let local = try LocalFileSystem(root: container.appendingPathComponent("files")) + for i in 0..<32 { try await local.createFile(path: WorkspacePath(normalizing: String(format: "/%02d", i)), data: Data([UInt8(i)])) } + for name in ["a/leaf", "a-", "a0/leaf", "a0-", "z"] { + try await local.createFile(path: WorkspacePath(normalizing: "/" + name), data: Data(name.utf8)) + } + let id = UUID() + var packBytes: [Data] = [] + var ranges: [Data] = [] + for (i, fs) in [CaptureSource(local, scoped: true), CaptureSource(local, scoped: false)].enumerated() { + let root = container.appendingPathComponent("store\(i)") + let store = FileCheckpointStore(rootDirectory: root) + _ = try await store.captureRevision(from: fs, draft: .init(workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + let packs = try artifacts(root.appendingPathComponent(id.uuidString).appendingPathComponent("packs")) + let pack = try #require(packs.first { $0.key.hasSuffix(".pack") }) + packBytes.append(pack.value) + let index = try #require(packs[String(pack.key.dropLast(5)) + ".json"]) + var entries = try #require(JSONSerialization.jsonObject(with: index) as? [String: [String: Any]]) + for hash in entries.keys { entries[hash]?.removeValue(forKey: "pack") } + ranges.append(try JSONSerialization.data(withJSONObject: entries, options: [.sortedKeys])) + } + #expect(packBytes[0] == packBytes[1]) + #expect(ranges[0] == ranges[1]) // random pack UUIDs differ; hashes and ranges must not. + } + + @Test + func unreadableDirectoriesAndFailedReadsPublishNoCheckpoint() async throws { + let container = try TestSupport.temporaryDirectory("CaptureErrors") + let files = container.appendingPathComponent("files") + let blocked = files.appendingPathComponent("blocked") + defer { _ = chmod(blocked.path, 0o755); TestSupport.remove(container) } + let local = try LocalFileSystem(root: files) + try await local.createFile(path: "/blocked/child", data: Data([1])) + for scoped in [false, true] { + let id = UUID() + let store = FileCheckpointStore(rootDirectory: container.appendingPathComponent("store")) + if geteuid() != 0 { + #expect(chmod(blocked.path, 0) == 0) + do { + _ = try await store.captureRevision(from: CaptureSource(local, scoped: scoped), draft: .init( + workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + Issue.record("unreadable directories must fail capture") + } catch { + #expect((error as NSError).domain == NSPOSIXErrorDomain) + #expect((error as NSError).code == Int(EACCES)) + } + #expect(chmod(blocked.path, 0o755) == 0) + } + // Disappearance after metadata, during the requested read, is an error + // on both routes and must not become a published partial snapshot. + let source = CaptureSource(local, scoped: scoped, failingRead: true) + do { + _ = try await store.captureRevision(from: source, draft: .init( + workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + Issue.record("failed content reads must fail capture") + } catch { #expect((error as NSError).code == Int(ENOENT)) } + #expect(try await store.listCheckpoints(workspaceId: id).isEmpty) + } + } + + private func artifacts(_ directory: URL) throws -> [String: Data] { + try Dictionary(uniqueKeysWithValues: FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .map { ($0.lastPathComponent, try Data(contentsOf: $0)) }) + } +} + +/// Records reads on the actual descriptor callback as well as the per-path route. +private final class CaptureSource: FileSystem, TraversingFileSystem, @unchecked Sendable { + let base: any FileSystem + let supportsScopedWalk: Bool + let failingRead: Bool + private let lock = NSLock() + private var recordedReads: [WorkspacePath] = [] + private var counts = (walk: 0, stat: 0, list: 0) + var reads: [WorkspacePath] { lock.withLock { recordedReads } } + var walkCount: Int { lock.withLock { counts.walk } } + var statCount: Int { lock.withLock { counts.stat } } + var listCount: Int { lock.withLock { counts.list } } + init(_ base: any FileSystem, scoped: Bool, failingRead: Bool = false) { + self.base = base; supportsScopedWalk = scoped; self.failingRead = failingRead + } + func reset() { lock.withLock { recordedReads = []; counts = (0, 0, 0) } } + private func record(_ path: WorkspacePath) throws { + lock.withLock { recordedReads.append(path) } + if failingRead { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } + } + nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { + lock.withLock { counts.walk += 1 } + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries) { info, read, link in + try await withoutActuallyEscaping(read) { read in + try await withoutActuallyEscaping(link) { link in + try await visit(info, { length in + try self.record(info.path) + return try await read(length) + }, { try await link() }) + } + } + } + } + func stat(path: WorkspacePath) async throws -> FileInfo { + lock.withLock { counts.stat += 1 } + return try await base.stat(path: path) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + lock.withLock { counts.list += 1 } + return try await base.listDirectory(path: path) + } + func readFile(path: WorkspacePath) async throws -> Data { try record(path); return try await base.readFile(path: path) } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { try await base.glob(pattern: pattern, currentDirectory: currentDirectory) } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { throw WorkspaceError.readOnly } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func remove(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { throw WorkspaceError.readOnly } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } +} + +/// Enumerates a child which disappears before its own metadata is requested. +private final class GhostFileSystem: FileSystem, Sendable { + let base: any FileSystem + let code: Int32 + init(base: any FileSystem, code: Int32) { self.base = base; self.code = code } + func stat(path: WorkspacePath) async throws -> FileInfo { + if path == "/ghost" { throw NSError(domain: NSPOSIXErrorDomain, code: Int(code)) } + return try await base.stat(path: path) + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + var entries = try await base.listDirectory(path: path) + if path.isRoot { entries.append(.init(name: "ghost", info: .init(path: "/ghost", kind: .file, size: 1, permissions: .defaultFile, modificationDate: nil))) } + return entries + } + func readFile(path: WorkspacePath) async throws -> Data { try await base.readFile(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { throw WorkspaceError.unsupported("test glob") } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { throw WorkspaceError.readOnly } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func remove(path: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { throw WorkspaceError.readOnly } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { throw WorkspaceError.readOnly } +} diff --git a/Tests/WorkspaceTests/CheckpointFormatTests.swift b/Tests/WorkspaceTests/CheckpointFormatTests.swift index 21f34f6..01e187a 100644 --- a/Tests/WorkspaceTests/CheckpointFormatTests.swift +++ b/Tests/WorkspaceTests/CheckpointFormatTests.swift @@ -325,6 +325,18 @@ struct CheckpointFormatTests { #expect(cache.cost == 0 && cache.count == 0) } + @Test func `capture cache inserts retain resident trees across an oversized scan`() { + var cache = CostLRU(limit: 8) + for key in 0..<7 { cache.insert(key, for: key, cost: 1) } + for key in 7..<100 { + cache.insert(key, for: key, cost: 1, evicting: .mostRecent) + #expect(cache.cost <= 8) + } + for key in 0..<7 { #expect(cache.value(for: key) == key) } + #expect(cache.value(for: 99) == 99) + #expect(cache.count == 8 && cache.cost == 8) + } + @Test func `durable orphan root is swept after metadata publication fails`() async throws { let root = try TestSupport.temporaryDirectory("OrphanRoot") defer { TestSupport.remove(root) } diff --git a/Tests/WorkspaceTests/FileSystemTraversalTests.swift b/Tests/WorkspaceTests/FileSystemTraversalTests.swift index 97afcd0..d9880da 100644 --- a/Tests/WorkspaceTests/FileSystemTraversalTests.swift +++ b/Tests/WorkspaceTests/FileSystemTraversalTests.swift @@ -105,6 +105,35 @@ struct FileSystemTraversalTests { print("DeepTreeLimitChecked") } + @Test(arguments: [false, true]) + func missingChildAtDescriptorFallbackHonorsWalkOption(skipVanished: Bool) async throws { + let root = try TestSupport.temporaryDirectory("WalkVanishedAtCap") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + let parent = WorkspacePath(normalizing: String(repeating: "/d", count: 31)) + let first = parent.appending("a"), vanished = parent.appending("b") + try await local.createFile(path: first, data: Data([1])) + try await local.createFile(path: vanished, data: Data([2])) + var visited: [WorkspacePath] = [] + do { + try await local.walk(skippingVanishedEntries: skipVanished) { info, read, _ in + visited.append(info.path) + if info.path == first { + #expect(try await read(nil) == Data([1])) + // Both siblings have already been enumerated/lstat'ed. The + // depth-32 handoff must tolerate a now-missing child lstat. + try await local.remove(path: vanished, recursive: false) + } + } + #expect(skipVanished) + } catch { + #expect(!skipVanished) + #expect((error as NSError).code == Int(ENOENT)) + } + #expect(visited.contains(first)) + #expect(!visited.contains(vanished)) + } + @Test func captureParityIncludesLinksModesAndSubtreeSpelling() async throws { let container = try TestSupport.temporaryDirectory("WalkParity") diff --git a/Tests/WorkspaceTests/TestSupport.swift b/Tests/WorkspaceTests/TestSupport.swift index 198c54c..2b2af8b 100644 --- a/Tests/WorkspaceTests/TestSupport.swift +++ b/Tests/WorkspaceTests/TestSupport.swift @@ -24,13 +24,15 @@ actor HookableFileSystem: FileSystem, ExactTimestampFileSystem, TraversingFileSy var walks: [WorkspacePath] = [] private let scopedWalk: Bool nonisolated var supportsScopedWalk: Bool { scopedWalk && base.supportsScopedWalk } - nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, + nonisolated(nonsending) func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, visit: FileSystemWalkVisitor) async throws { guard supportsScopedWalk else { - return try await walkUsingPaths(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + return try await walkUsingPaths(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } await recordWalk(path) - try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, visit: visit) + try await base.walk(at: path, skippingUnreadableDirectories: skippingUnreadableDirectories, + skippingVanishedEntries: skippingVanishedEntries, visit: visit) } private func recordWalk(_ path: WorkspacePath) { walks.append(path) } func resetMetadataCounts() { walks = []; statCount = 0; listings = [] } From 815be72aab453d3d29289333966fe6566855f563 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 06:04:24 -0700 Subject: [PATCH 18/34] Make restore and checkpoint diffs scale with the change restore(to:) and diff(from: .checkpoint, to: .current) no longer capture the whole current tree. They compare the checkpoint's persisted capture metadata with one bulk metadata walk of the current tree, applying the same racy-mtime and identity rules as transactions, and read and hash only entries whose metadata differs or can't be trusted. Restore writes only the differences, records sparse before/after snapshots for history and failure recovery, and publishes the restored checkpoint by reusing stored blobs and trees. diff(.current, .current) returns without touching the filesystem. Snapshot capture keeps a per-entry stat on its per-path route, so AuthorizedFileSystem's per-entry checks still apply, and InMemoryFileSystem gains a native actor-owned bulk walk that takes the fast route instead. On the medium benchmark fixture a directory-persistence restore drops from about 1.3 s to 106 ms (reading 12 KiB instead of 604 MiB), and a checkpoint diff from about 310 ms to 89 ms. --- README.md | 20 +- .../Workspace/CheckpointStore+Revision.swift | 205 +++++++++++ Sources/Workspace/CheckpointStore.swift | 101 +++--- Sources/Workspace/FS/InMemoryFileSystem.swift | 100 ++++- Sources/Workspace/RevisionComparison.swift | 192 ++++++++++ Sources/Workspace/Snapshot.swift | 101 +++--- Sources/Workspace/Workspace+Checkpoints.swift | 60 ++- Sources/Workspace/Workspace+Internals.swift | 19 +- .../Workspace+TransactionStorage.swift | 13 +- Sources/Workspace/Workspace.swift | 4 +- Sources/Workspace/WorkspaceRevision.swift | 26 +- .../CheckpointCaptureTraversalTests.swift | 106 +++++- .../FileSystemTraversalTests.swift | 73 ++++ .../InMemoryTraversalTests.swift | 146 ++++++++ Tests/WorkspaceTests/RestoreDiffTests.swift | 341 ++++++++++++++++++ 15 files changed, 1357 insertions(+), 150 deletions(-) create mode 100644 Sources/Workspace/CheckpointStore+Revision.swift create mode 100644 Sources/Workspace/RevisionComparison.swift create mode 100644 Tests/WorkspaceTests/InMemoryTraversalTests.swift create mode 100644 Tests/WorkspaceTests/RestoreDiffTests.swift diff --git a/README.md b/README.md index 6a670fe..1f975b6 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,8 @@ metadata and read changed or unverifiable files; previews and merges materialize Metadata capture uses bulk traversal when available and per-path enumeration otherwise, including scoped subtree refreshes. Mounted traversal preserves virtual paths and shadowing; authorized traversal retains per-entry permission checks. +Snapshot capture likewise stats each entry on its per-path route. InMemoryFileSystem has a native +actor-owned traversal; after reentrant mutations it resolves pending paths and reads again. Checkpoint persistence still requires the store's metadata and manifest work. Checkpoints prime a transaction base only after that instance has begun a transaction. Directory commits replace the base with the new revision index, without retaining changed-file contents or revision read caches. @@ -307,7 +309,23 @@ and the previous mtime is more than two seconds older than the start of the prev capture. Identity must also match when both entries provide it. Missing mtimes and recent timestamps require reads; InMemoryFileSystem's exact timestamps skip this racy window. Writes that preserve safely old metadata can evade the check. Use `.verifyContents` to read and compare every file during -transaction begin and commit, and to read and hash every file during directory checkpoint capture. +transaction begin and commit, checkpoint-to-current diff and restore, and to read and hash every +file during directory checkpoint capture. + +Checkpoint-to-current diffs and `restore(to:)` use one bulk metadata walk and compare against the +checkpoint's persisted capture hints and revision hashes. They read only suspect files (changed +metadata, missing hints, or timestamps inside the same two-second window). Identity is compared +when both captures provide it; exact-timestamp backends skip the window. Older revisions without +capture hints conservatively require content reads. Memory persistence retains hints for only the +four most recently captured or restored revisions; older revisions also fall back to content reads. +As with transactions, writes that preserve +safely old metadata can evade `.trustMetadata`; use `.verifyContents` when those writes must be +detected and restored. + +Restore applies and recaptures only differing entries and necessary ancestor permissions. Its +failure recovery and recorded changes use the same sparse scope. The new checkpoint reuses existing +blobs and directory trees through the store's normal locked, durable publication path. Archive +restore continues to capture its selected subtree in full. Transactions are logically atomic within cooperative workspace use. They are not crash-safe against an external process racing the commit. In-process `Workspace.compact` protects revisions referenced diff --git a/Sources/Workspace/CheckpointStore+Revision.swift b/Sources/Workspace/CheckpointStore+Revision.swift new file mode 100644 index 0000000..cb15a3d --- /dev/null +++ b/Sources/Workspace/CheckpointStore+Revision.swift @@ -0,0 +1,205 @@ +import Foundation + +/// Separate from the capture walker: readers obtain a frozen index and its capture +/// hints together; writers publish a sparse patch over an existing immutable revision. +protocol CheckpointRevisionStore: Sendable { + func loadCapturedRevision(id: UUID, workspaceId: UUID) async throws -> CapturedRevision? + func publishRevision( + reusing id: UUID, patch: Snapshot, preservingDirectories: Set, + metadata: MetadataIndex, draft: CheckpointDraft + ) async throws -> Checkpoint +} + +struct CapturedRevision: Sendable { + var index: RevisionIndex + var metadata: MetadataIndex +} + +/// Hints are optional: evicted revisions use content verification. Bound retention +/// across workspaces so checkpoint history does not retain a second full tree each time. +struct CaptureMetadataCache: Sendable { + private var entries: [UUID: MetadataIndex] = [:] + private var recent: [UUID] = [] + + subscript(id: UUID) -> MetadataIndex? { entries[id] } + + mutating func insert(_ metadata: MetadataIndex, for id: UUID) { + recent.removeAll { $0 == id } + recent.append(id) + entries[id] = metadata + if recent.count > 4 { entries.removeValue(forKey: recent.removeFirst()) } + } + + mutating func retain(_ ids: Set) { + entries = entries.filter { ids.contains($0.key) } + recent.removeAll { !ids.contains($0) } + } +} + +extension CheckpointStore { + // Conservative compatibility for decorators and stores without capture hints. + func loadCapturedRevision(id: UUID, workspaceId: UUID) async throws -> CapturedRevision? { + guard let index = try await loadRevisionIndex(id: id, workspaceId: workspaceId) else { return nil } + return CapturedRevision(index: index, metadata: MetadataIndex()) + } + + func publishRevision( + reusing id: UUID, patch: Snapshot, preservingDirectories: Set, + metadata: MetadataIndex, draft: CheckpointDraft + ) async throws -> Checkpoint { + guard var snapshot = try await loadSnapshot(id: id, workspaceId: draft.workspaceID) else { + throw WorkspaceError.revisionDataNotFound(id) + } + snapshot.id = UUID() + snapshot.entry = Snapshot.patching(snapshot.entry, with: patch.entry, preserving: preservingDirectories) + return try await saveRevision(snapshot, draft: draft) + } +} + +extension InMemoryCheckpointStore { + func captureRevisionWithMetadata(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint { + let cursor = persistedMutationCursor(workspaceId: draft.workspaceID) + var metadata: MetadataIndex? = MetadataIndex() + let snapshot = try await Snapshot.capture(from: filesystem, metadata: &metadata) + let checkpoint = try persistRevision(snapshot, draft: draft, mutationCursor: cursor) + captureMetadataBySnapshot.insert(metadata!, for: snapshot.id) + return checkpoint + } + + func loadCapturedRevision(id: UUID, workspaceId: UUID) async throws -> CapturedRevision? { + guard let index = try await loadRevisionIndex(id: id, workspaceId: workspaceId) else { return nil } + return CapturedRevision(index: index, metadata: captureMetadataBySnapshot[id] ?? MetadataIndex()) + } + + func publishRevision( + reusing id: UUID, patch: Snapshot, preservingDirectories: Set, + metadata: MetadataIndex, draft: CheckpointDraft + ) async throws -> Checkpoint { + guard var snapshot = snapshotsByWorkspace[draft.workspaceID]?[id] else { + throw WorkspaceError.revisionDataNotFound(id) + } + snapshot.id = UUID() + snapshot.entry = Snapshot.patching(snapshot.entry, with: patch.entry, preserving: preservingDirectories) + let checkpoint = try persistRevision(snapshot, draft: draft, + mutationCursor: persistedMutationCursor(workspaceId: draft.workspaceID)) + captureMetadataBySnapshot.insert(metadata, for: snapshot.id) + return checkpoint + } +} + +extension FileCheckpointStore { + func loadCapturedRevision(id: UUID, workspaceId: UUID) async throws -> CapturedRevision? { + try await validateWorkspaceFormatIfPresent(workspaceId) + guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { return nil } + return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { + guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { return nil } + var metadata = MetadataIndex() + // Legacy materialized snapshots have no trustworthy capture timestamp. + metadata.capturedAt = manifest.captureStartedAt ?? .distantPast + metadata.exactTimestamps = manifest.exactContentTimestamps ?? false + func visit(_ node: SnapshotManifest.Node) throws -> RevisionIndex.Entry { + let entry = try revisionEntry(from: node, workspaceId: workspaceId, depth: 0) + if let kind = entry.kind { + metadata.entries[node.path] = .init(info: FileInfo( + path: node.path, kind: kind, size: entry.size, + permissions: entry.permissions!, modificationDate: node.modificationDate, identity: node.identity + ), target: node.symlinkTarget) + } + if case let .directory(path, permissions, _) = entry { + return .directory(path: path, permissions: permissions, + children: try children(of: node, workspaceId: workspaceId).map(visit)) + } + return entry + } + let entry = try visit(manifest.root) + return CapturedRevision(index: RevisionIndex(id: id, root: manifest.rootPath, entry: entry), metadata: metadata) + } + } + + func publishRevision( + reusing id: UUID, patch: Snapshot, preservingDirectories: Set, + metadata: MetadataIndex, draft: CheckpointDraft + ) async throws -> Checkpoint { + try await ensureWorkspaceDirectories(for: draft.workspaceID) + return try await withExclusiveLock(at: lifecycleLockURL(workspaceId: draft.workspaceID)) { + let workspaceId = draft.workspaceID + let cursor = try await persistedMutationCursor(workspaceId: workspaceId) + guard let source = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { + throw WorkspaceError.revisionDataNotFound(id) + } + let parent = resolvedParent(for: draft, checkpoints: try loadAllCheckpointsUnlocked(workspaceId: workspaceId)) + let parentManifest = try parent.map { + guard let manifest = try loadManifestUnlocked(id: $0.snapshotId, workspaceId: workspaceId) else { + throw WorkspaceError.revisionDataNotFound($0.snapshotId) + } + return manifest + } + try requireCurrentFormatUnlocked(workspaceId: workspaceId) + let batch = try makeBlobBatch(workspaceId: workspaceId) + defer { batch.close() } + func apply(_ patch: Snapshot.Entry, to old: SnapshotManifest.Node?) throws -> SnapshotManifest.Node { + if case let .directory(directory) = patch { + let previous: [SnapshotManifest.Node] + if let old { previous = try children(of: old, workspaceId: workspaceId) } + else { previous = [] } + let byPath = Dictionary(uniqueKeysWithValues: previous.map { ($0.path, $0) }) + var entries = preservingDirectories.contains(directory.path) ? byPath : [:] + for child in directory.children { + if case .missing = child { entries.removeValue(forKey: child.path) } + else { entries[child.path] = try apply(child, to: byPath[child.path]) } + } + let updated = entries.values.sorted { $0.path < $1.path } + var node = SnapshotManifest.Node(kind: .directory, path: directory.path, + permissions: directory.permissions, + modificationDate: metadata.entries[directory.path]?.info.modificationDate, + identity: metadata.entries[directory.path]?.info.identity) + if let old, old.kind == .directory, old.treeHash != nil, updated == previous { + node.treeHash = old.treeHash + return node + } + node.children = updated + return try persistDirectory(node, workspaceId: workspaceId) + } + var node = try writeManifestNode(patch, workspaceId: workspaceId, batch: batch, reusing: old) + node.modificationDate = metadata.entries[patch.path]?.info.modificationDate + node.identity = metadata.entries[patch.path]?.info.identity + return node + } + let root = try apply(patch.entry, to: source.root) + try finish(batch, workspaceId: workspaceId) + let manifest = SnapshotManifest(version: 3, id: UUID(), rootPath: source.rootPath, + root: root, captureStartedAt: metadata.capturedAt, + exactContentTimestamps: metadata.exactTimestamps) + let summary: ChangeSet.Summary + if let parentManifest { + summary = try await compareNodes(parentManifest.root, root, workspaceId: workspaceId).summary + } else { + summary = try await compareNodes(.init(kind: .missing, path: source.rootPath), root, workspaceId: workspaceId).summary + } + let checkpoint = Checkpoint.make(snapshotID: manifest.id, draft: draft, mutationCursor: cursor, + parent: parent, summary: summary) + try Task.checkCancellation() + // All immutable objects are durable before the manifest, then the checkpoint + // is published through the same full-sync/fallback path as captureRevision. + try write(manifest, to: snapshotURL(id: manifest.id, workspaceId: workspaceId), durability: .publication) + try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: workspaceId), durability: .publication) + listCheckpointsCache[workspaceId] = nil + return checkpoint + } + } +} + +extension Snapshot { + static func patching(_ old: Entry, with patch: Entry, preserving directories: Set) -> Entry { + guard case let .directory(new) = patch, directories.contains(new.path), + case let .directory(previous) = old else { return patch } + var children = Dictionary(uniqueKeysWithValues: previous.children.map { ($0.path, $0) }) + for child in new.children { + if case .missing = child { children.removeValue(forKey: child.path) } + else { children[child.path] = patching(children[child.path] ?? .missing(.init(path: child.path)), + with: child, preserving: directories) } + } + return .directory(.init(path: new.path, permissions: new.permissions, + children: children.values.sorted { $0.path < $1.path })) + } +} diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 575584e..e85c0e4 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -11,7 +11,7 @@ import Glibc /// The store is the **source of truth** for `Mutation.sequence` values. Callers may pass /// any placeholder sequence; ``appendMutation(_:)`` returns the record with the next persisted /// monotonic number for that workspace, serialized under the mutations lock. -protocol CheckpointStore: AnyObject, Sendable { +protocol CheckpointStore: CheckpointRevisionStore, AnyObject, Sendable { func prepare(workspaceId: UUID) async throws func changes(workspaceId: UUID) async throws -> AsyncStream func captureRevision(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint @@ -75,7 +75,8 @@ extension CheckpointStore { /// An in-memory checkpoint store for tests and ephemeral workspaces. actor InMemoryCheckpointStore: CheckpointStore { private var checkpointsByWorkspace: [UUID: [Checkpoint]] = [:] - private var snapshotsByWorkspace: [UUID: [UUID: Snapshot]] = [:] + var snapshotsByWorkspace: [UUID: [UUID: Snapshot]] = [:] + var captureMetadataBySnapshot = CaptureMetadataCache() private(set) var revisionIndexBuildCount = 0 private var revisionIndexesByWorkspace: [UUID: [UUID: RevisionIndex]] = [:] private var mutationsByWorkspace: [UUID: [Mutation]] = [:] @@ -92,20 +93,18 @@ actor InMemoryCheckpointStore: CheckpointStore { } func captureRevision(from filesystem: any FileSystem, draft: CheckpointDraft) async throws -> Checkpoint { - let mutationCursor = persistedMutationCursor(workspaceId: draft.workspaceID) - let snapshot = try await Snapshot.capture(from: filesystem) - return try persistRevision(snapshot, draft: draft, mutationCursor: mutationCursor) + try await captureRevisionWithMetadata(from: filesystem, draft: draft) } func saveRevision(_ snapshot: Snapshot, draft: CheckpointDraft) async throws -> Checkpoint { try persistRevision(snapshot, draft: draft, mutationCursor: persistedMutationCursor(workspaceId: draft.workspaceID)) } - private func persistedMutationCursor(workspaceId: UUID) -> Int { + func persistedMutationCursor(workspaceId: UUID) -> Int { mutationHighWaterMarks[workspaceId] ?? 0 } - private func persistRevision(_ snapshot: Snapshot, draft: CheckpointDraft, mutationCursor: Int) throws -> Checkpoint { + func persistRevision(_ snapshot: Snapshot, draft: CheckpointDraft, mutationCursor: Int) throws -> Checkpoint { var list = checkpointsByWorkspace[draft.workspaceID] ?? [] let parent = resolvedParent(for: draft, checkpoints: list) let parentSnapshot = parent.flatMap { snapshotsByWorkspace[draft.workspaceID]?[$0.snapshotId] } @@ -270,6 +269,10 @@ actor InMemoryCheckpointStore: CheckpointStore { retainedSnapshotIDs.contains($0.key) } } + if !dryRun { + let live = Set(snapshotsByWorkspace.values.flatMap { $0.keys }) + captureMetadataBySnapshot.retain(live) + } return StoreCompactionResult(report: report, checkpoints: retained) } @@ -329,7 +332,7 @@ actor InMemoryCheckpointStore: CheckpointStore { /// application-level serialization. actor FileCheckpointStore: CheckpointStore { private let rootDirectory: URL - private let fileManager: FileManager + let fileManager: FileManager private let encoder: JSONEncoder private let decoder: JSONDecoder private let compactEncoder: JSONEncoder @@ -339,7 +342,7 @@ actor FileCheckpointStore: CheckpointStore { private var heldLocks: Set = [] private var lockWaiters: [PathName: [CheckedContinuation]] = [:] - private var listCheckpointsCache: [UUID: (cacheKey: String, checkpoints: [Checkpoint])] = [:] + var listCheckpointsCache: [UUID: (cacheKey: String, checkpoints: [Checkpoint])] = [:] init( rootDirectory: URL, @@ -441,7 +444,7 @@ actor FileCheckpointStore: CheckpointStore { /// Schemas 1/2 inline the recursive tree and reference loose blobs. Schema 3 /// stores a small root, content-addressed directory references, and optional pack ranges. /// The outer store format is version 2, fencing writers that only understand inline trees. - private struct SnapshotManifest: Codable, Sendable { + struct SnapshotManifest: Codable, Sendable { struct Node: Codable, Sendable, Equatable { enum Kind: String, Codable { case file, directory, symlink, missing @@ -453,6 +456,7 @@ actor FileCheckpointStore: CheckpointStore { var contentHash: String? var contentSize: UInt64? = nil var modificationDate: Date? = nil + var identity: FileIdentity? = nil var symlinkTarget: String? var children: [Node]? var treeHash: String? = nil @@ -461,7 +465,7 @@ actor FileCheckpointStore: CheckpointStore { static func == (lhs: Self, rhs: Self) -> Bool { lhs.kind == rhs.kind && lhs.path == rhs.path && lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && - lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && + lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && lhs.identity == rhs.identity && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && lhs.children == rhs.children && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } @@ -473,17 +477,19 @@ actor FileCheckpointStore: CheckpointStore { var root: Node // Older manifests and materialized snapshots have no trustworthy capture timestamp. var captureStartedAt: Date? = nil + var exactContentTimestamps: Bool? = nil private enum CodingKeys: String, CodingKey { - case version, id, rootPath, root, treeRoot, captureStartedAt + case version, id, rootPath, root, treeRoot, captureStartedAt, exactContentTimestamps } - init(version: Int, id: UUID, rootPath: WorkspacePath, root: Node, captureStartedAt: Date? = nil) { + init(version: Int, id: UUID, rootPath: WorkspacePath, root: Node, captureStartedAt: Date? = nil, exactContentTimestamps: Bool? = nil) { self.version = version self.id = id self.rootPath = rootPath self.root = root self.captureStartedAt = captureStartedAt + self.exactContentTimestamps = exactContentTimestamps } init(from decoder: any Decoder) throws { @@ -493,6 +499,7 @@ actor FileCheckpointStore: CheckpointStore { rootPath = try values.decode(WorkspacePath.self, forKey: .rootPath) root = try values.decode(Node.self, forKey: version >= 3 ? .treeRoot : .root) captureStartedAt = try values.decodeIfPresent(Date.self, forKey: .captureStartedAt) + exactContentTimestamps = try values.decodeIfPresent(Bool.self, forKey: .exactContentTimestamps) } func encode(to encoder: any Encoder) throws { @@ -504,6 +511,7 @@ actor FileCheckpointStore: CheckpointStore { // legacy operation must fail decoding before it can sweep live blobs. try values.encode(root, forKey: version >= 3 ? .treeRoot : .root) try values.encodeIfPresent(captureStartedAt, forKey: .captureStartedAt) + try values.encodeIfPresent(exactContentTimestamps, forKey: .exactContentTimestamps) } } @@ -515,6 +523,7 @@ actor FileCheckpointStore: CheckpointStore { var contentHash: String? var contentSize: UInt64? var modificationDate: Date? + var identity: FileIdentity? var symlinkTarget: String? var treeHash: String? var packed: PackedBlob? @@ -533,6 +542,7 @@ actor FileCheckpointStore: CheckpointStore { contentHash = node.contentHash contentSize = node.contentSize modificationDate = node.modificationDate + identity = node.identity symlinkTarget = node.symlinkTarget treeHash = node.treeHash packed = node.packed @@ -551,14 +561,14 @@ actor FileCheckpointStore: CheckpointStore { func node(at path: WorkspacePath) -> SnapshotManifest.Node { .init(kind: kind, path: path, permissions: permissions, - contentHash: contentHash, contentSize: contentSize, modificationDate: modificationDate, + contentHash: contentHash, contentSize: contentSize, modificationDate: modificationDate, identity: identity, symlinkTarget: symlinkTarget, treeHash: treeHash, packed: packed) } static func == (lhs: Self, rhs: Self) -> Bool { lhs.name.utf8.elementsEqual(rhs.name.utf8) && lhs.kind == rhs.kind && lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && - lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && + lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && lhs.identity == rhs.identity && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } @@ -618,7 +628,8 @@ actor FileCheckpointStore: CheckpointStore { ) try self.finish(batch, workspaceId: draft.workspaceID) let manifest = SnapshotManifest( - version: 3, id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt + version: 3, id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt, + exactContentTimestamps: await filesystem.supportsExactContentTimestamps ) let summary: ChangeSet.Summary if let parentManifest { @@ -659,7 +670,7 @@ actor FileCheckpointStore: CheckpointStore { try write(manifest, to: snapshotURL(id: snapshot.id, workspaceId: workspaceId), durability: .publication) } - private func loadManifestUnlocked(id: UUID, workspaceId: UUID) throws -> SnapshotManifest? { + func loadManifestUnlocked(id: UUID, workspaceId: UUID) throws -> SnapshotManifest? { let url = snapshotURL(id: id, workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return nil } let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) @@ -798,10 +809,11 @@ actor FileCheckpointStore: CheckpointStore { var reused = previous reused.permissions = info.permissions reused.modificationDate = info.modificationDate + reused.identity = info.identity return reused } return try persistDirectory(SnapshotManifest.Node( - kind: .directory, path: info.path, permissions: info.permissions, modificationDate: info.modificationDate + kind: .directory, path: info.path, permissions: info.permissions, modificationDate: info.modificationDate, identity: info.identity ), entries: entries, workspaceId: workspaceId, capturing: true) } @@ -857,6 +869,7 @@ actor FileCheckpointStore: CheckpointStore { modificationDate.addingTimeInterval(2) < previousCaptureStartedAt, previous?.kind == .file, previous?.contentSize == info.size, + (previous?.identity == nil || info.identity == nil || previous?.identity == info.identity), previous?.modificationDate != nil, previous?.modificationDate == info.modificationDate, let hash = previous?.contentHash, @@ -869,7 +882,7 @@ actor FileCheckpointStore: CheckpointStore { permissions: info.permissions, contentHash: hash, contentSize: info.size, - modificationDate: info.modificationDate, + modificationDate: info.modificationDate, identity: info.identity, packed: previous.packed ) } @@ -886,7 +899,7 @@ actor FileCheckpointStore: CheckpointStore { permissions: info.permissions, contentHash: hash, contentSize: UInt64(data.count), - modificationDate: info.modificationDate, + modificationDate: info.modificationDate, identity: info.identity, packed: packed ) case .symlink: @@ -894,7 +907,7 @@ actor FileCheckpointStore: CheckpointStore { kind: .symlink, path: path, permissions: info.permissions, - modificationDate: info.modificationDate, + modificationDate: info.modificationDate, identity: info.identity, symlinkTarget: try await readLink() ) case .directory: @@ -943,7 +956,7 @@ actor FileCheckpointStore: CheckpointStore { } } - private func revisionEntry( + func revisionEntry( from node: SnapshotManifest.Node, workspaceId: UUID, depth: Int? = nil, @@ -1000,7 +1013,7 @@ actor FileCheckpointStore: CheckpointStore { return node } - private func writeManifestNode( + func writeManifestNode( _ entry: Snapshot.Entry, workspaceId: UUID, batch: BlobBatch, @@ -1337,7 +1350,7 @@ actor FileCheckpointStore: CheckpointStore { return StoreCompactionResult(report: report, checkpoints: retained) } - private func loadAllCheckpointsUnlocked(workspaceId: UUID) throws -> [Checkpoint] { + func loadAllCheckpointsUnlocked(workspaceId: UUID) throws -> [Checkpoint] { let directory = checkpointsDirectoryURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: directory.path) else { return [] } return try artifactURLs(in: directory, pathExtension: "json").map { url in @@ -1360,7 +1373,7 @@ actor FileCheckpointStore: CheckpointStore { ) } - private func resolvedParent(for draft: CheckpointDraft, checkpoints: [Checkpoint]) -> Checkpoint? { + func resolvedParent(for draft: CheckpointDraft, checkpoints: [Checkpoint]) -> Checkpoint? { if let preferred = draft.preferredParentID, let parent = checkpoints.first(where: { $0.id == preferred }) { return parent @@ -1637,7 +1650,7 @@ actor FileCheckpointStore: CheckpointStore { /// Call while holding the lifecycle lock. The cursor is authoritative even when the /// Workspace that requested the checkpoint has not refreshed its mutation cache. - private func persistedMutationCursor(workspaceId: UUID) async throws -> Int { + func persistedMutationCursor(workspaceId: UUID) async throws -> Int { try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { try mutationSequenceState(workspaceId: workspaceId).highWaterMark } @@ -1761,7 +1774,7 @@ actor FileCheckpointStore: CheckpointStore { try fileManager.createDirectory(at: url, withIntermediateDirectories: true) } - private func ensureWorkspaceDirectories(for workspaceId: UUID) async throws { + func ensureWorkspaceDirectories(for workspaceId: UUID) async throws { try ensureDirectory(rootDirectory) try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { let version = try ensureWorkspaceDirectoriesLocked(for: workspaceId) @@ -1806,7 +1819,7 @@ actor FileCheckpointStore: CheckpointStore { return version } - private func validateWorkspaceFormatIfPresent(_ workspaceId: UUID) async throws { + func validateWorkspaceFormatIfPresent(_ workspaceId: UUID) async throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: workspaceURL.path) else { return } try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { @@ -1827,7 +1840,7 @@ actor FileCheckpointStore: CheckpointStore { return format.version } - private func workspaceDirectoryURL(workspaceId: UUID) -> URL { + func workspaceDirectoryURL(workspaceId: UUID) -> URL { rootDirectory.appendingPathComponent(workspaceId.uuidString, isDirectory: true) } @@ -1839,11 +1852,11 @@ actor FileCheckpointStore: CheckpointStore { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("snapshots", isDirectory: true) } - private func checkpointURL(id: UUID, workspaceId: UUID) -> URL { + func checkpointURL(id: UUID, workspaceId: UUID) -> URL { checkpointsDirectoryURL(workspaceId: workspaceId).appendingPathComponent("\(id.uuidString).json", isDirectory: false) } - private func snapshotURL(id: UUID, workspaceId: UUID) -> URL { + func snapshotURL(id: UUID, workspaceId: UUID) -> URL { snapshotsDirectoryURL(workspaceId: workspaceId).appendingPathComponent("\(id.uuidString).json", isDirectory: false) } @@ -1914,7 +1927,7 @@ actor FileCheckpointStore: CheckpointStore { } /// Serializes checkpoint, manifest, and blob lifecycle operations across store instances. - private func lifecycleLockURL(workspaceId: UUID) -> URL { + func lifecycleLockURL(workspaceId: UUID) -> URL { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("lifecycle.lock", isDirectory: false) } @@ -1922,9 +1935,9 @@ actor FileCheckpointStore: CheckpointStore { rootDirectory.appendingPathComponent(".\(workspaceId.uuidString).format.lock", isDirectory: false) } - private enum WriteDurability { case atomic, publication } + enum WriteDurability { case atomic, publication } - private func write( + func write( _ value: T, to url: URL, durability: WriteDurability ) throws { let data = try encoder.encode(value) @@ -1956,7 +1969,7 @@ actor FileCheckpointStore: CheckpointStore { private var packCatalogWorkspace: UUID? private var packCatalogKey: String? - private struct PackedBlob: Codable, Sendable, Equatable { + struct PackedBlob: Codable, Sendable, Equatable { var pack: String var offset: UInt64 var size: UInt64 @@ -1967,7 +1980,7 @@ actor FileCheckpointStore: CheckpointStore { /// One writer, owned by one lifecycle-locked operation. Packs are capped at 16 MiB /// (except a single large blob), with a 1 MiB write buffer. The first 32 unique /// blobs stay loose so tiny checkpoints don't require pack/index overhead. - private final class BlobBatch { + final class BlobBatch { let loose: URL let packs: URL var known: CostLRU @@ -2079,7 +2092,7 @@ actor FileCheckpointStore: CheckpointStore { try Self.durableWrite(Data(UUID().uuidString.utf8), to: url) } - private func makeBlobBatch(workspaceId: UUID) throws -> BlobBatch { + func makeBlobBatch(workspaceId: UUID) throws -> BlobBatch { let packs = packsDirectoryURL(workspaceId: workspaceId) let key = try packGeneration(workspaceId: workspaceId) var known = packCatalog @@ -2103,7 +2116,7 @@ actor FileCheckpointStore: CheckpointStore { return BlobBatch(loose: loose, packs: packs, known: known, fileManager: fileManager, encoder: compactEncoder) } - private func finish(_ batch: BlobBatch, workspaceId: UUID) throws { + func finish(_ batch: BlobBatch, workspaceId: UUID) throws { try batch.finishPack() if batch.wrotePacks { try advancePackGeneration(workspaceId: workspaceId) } packCatalog = batch.known @@ -2118,7 +2131,7 @@ actor FileCheckpointStore: CheckpointStore { treeCache.insert(entries, for: key, cost: cost, evicting: capturing ? .mostRecent : .leastRecent) } - private func children(of node: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) throws -> [SnapshotManifest.Node] { + func children(of node: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) throws -> [SnapshotManifest.Node] { guard node.kind == .directory else { return [] } guard let hash = node.treeHash else { return (node.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) } } return try treeEntries(hash: hash, workspaceId: workspaceId, capturing: capturing).map { $0.node(in: node.path) } @@ -2148,7 +2161,7 @@ actor FileCheckpointStore: CheckpointStore { return entries } - private func persistDirectory( + func persistDirectory( _ node: SnapshotManifest.Node, workspaceId: UUID, syncParent: Bool = false ) throws -> SnapshotManifest.Node { let entries = try (node.children ?? []).sorted { @@ -2316,7 +2329,7 @@ actor FileCheckpointStore: CheckpointStore { statistics: .init(changedPathCount: paths.count, changedFileCount: files, additions: additions, deletions: 0)) } - private func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) async throws -> ChangeSet { + func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) async throws -> ChangeSet { if old == new { return ChangeSet(changes: []) } if old.kind == .directory, new.kind == .directory { var changes: [FileChange] = (old.permissions ?? .defaultDirectory) == (new.permissions ?? .defaultDirectory) ? [] : [.init(path: new.path, kind: .directory, effect: .modified)] @@ -2350,7 +2363,7 @@ actor FileCheckpointStore: CheckpointStore { return try readBlob(node, workspaceId: workspaceId) } - private func requireCurrentFormatUnlocked(workspaceId: UUID) throws { + func requireCurrentFormatUnlocked(workspaceId: UUID) throws { let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("format.json") guard try read(StoreFormat.self, from: url).version == StoreFormat.currentVersion else { throw WorkspaceError.storageCorrupted("store format changed after preparation") @@ -2448,11 +2461,11 @@ actor FileCheckpointStore: CheckpointStore { private enum LockMode: Sendable { case shared, exclusive } - private func withExclusiveLock(at url: URL, _ body: () async throws -> R) async throws -> R { + func withExclusiveLock(at url: URL, _ body: () async throws -> R) async throws -> R { try await withLock(at: url, mode: .exclusive, body) } - private func withSharedLock(at url: URL, _ body: () async throws -> R) async throws -> R { + func withSharedLock(at url: URL, _ body: () async throws -> R) async throws -> R { try await withLock(at: url, mode: .shared, body) } diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index 1221666..6f029ad 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -13,7 +13,7 @@ import Glibc /// /// Actor isolation serializes access to the tree so the filesystem can be shared safely across /// concurrent tasks. -public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, ExactTimestampFileSystem { +public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, ExactTimestampFileSystem, TraversingFileSystem { package nonisolated var hasExactContentTimestamps: Bool { true } private var lastTimestamp = Date().timeIntervalSinceReferenceDate @@ -87,6 +87,104 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E root = makeNode(kind: .directory([:]), permissions: 0o755) } + // Cursor frames and Node references stay on this actor. Visitors remain on their + // caller's executor; only immutable metadata/content values cross that boundary. + private final class WalkCursor { + enum Frame { + case entry(WorkspacePath, Node?, TimeInterval?) + case directory(FileInfo) + } + var frames: [Frame] + init(path: WorkspacePath) { frames = [.entry(path, nil, nil)] } + } + private struct WalkEntry: Sendable { + var info: FileInfo + var capturedAt: TimeInterval + var data: Data? + var target: String? + } + private var walkCursors: [UUID: WalkCursor] = [:] + + nonisolated(nonsending) package func walk( + at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, + visit: FileSystemWalkVisitor + ) async throws { + let id = await beginWalk(at: path) + do { + while let entry = try await nextWalkEntry(id, skippingVanishedEntries: skippingVanishedEntries) { + try await visit(entry.info, { length in + try await self.readWalkFile(at: entry.info.path, capturedAt: entry.capturedAt, + data: entry.data, length: length) + }, { + try await self.readWalkLink(at: entry.info.path, capturedAt: entry.capturedAt, target: entry.target) + }) + } + await endWalk(id) + } catch { + await endWalk(id) + throw error + } + } + + private func beginWalk(at path: WorkspacePath) -> UUID { + let id = UUID() + walkCursors[id] = WalkCursor(path: path) + return id + } + + private func endWalk(_ id: UUID) { walkCursors.removeValue(forKey: id) } + + private func nextWalkEntry(_ id: UUID, skippingVanishedEntries: Bool) throws -> WalkEntry? { + let cursor = walkCursors[id]! + while let frame = cursor.frames.popLast() { + try Task.checkCancellation() + switch frame { + case let .directory(info): + return WalkEntry(info: info, capturedAt: lastTimestamp) + case let .entry(path, listed, listedAt): + let entry: Node + do { + // Every mutation advances lastTimestamp before yielding. After + // reentrancy, resolve the listed path again, never detached nodes. + if let listed, listedAt == lastTimestamp { entry = listed } + else { entry = try node(at: path, followFinalSymlink: false) } + } catch { + guard (listed == nil || skippingVanishedEntries), Snapshot.isMissingEntry(error) else { throw error } + continue + } + let info = fileInfo(for: entry, path: path) + switch entry.kind { + case let .directory(children): + cursor.frames.append(.directory(info)) + let ordered = children.map { name, child in + (name, child, walkSortKey(name: name.string, isDirectory: child.isDirectory)) + }.sorted { $0.2.utf8.lexicographicallyPrecedes($1.2.utf8) } + for (name, child, _) in ordered.reversed() { + cursor.frames.append(.entry(path.appending(name.string), child, lastTimestamp)) + } + case let .file(data): + return WalkEntry(info: info, capturedAt: lastTimestamp, data: data) + case let .symlink(target): + return WalkEntry(info: info, capturedAt: lastTimestamp, target: target) + } + } + } + return nil + } + + private func readWalkFile(at path: WorkspacePath, capturedAt: TimeInterval, data: Data?, length: Int?) async throws -> Data { + if let length, length < 0 { throw WorkspaceError.unsupported("read length must not be negative") } + guard capturedAt == lastTimestamp, let data else { + return try await readFile(path: path, offset: 0, length: length) + } + return length.map { Data(data.prefix($0)) } ?? data + } + + private func readWalkLink(at path: WorkspacePath, capturedAt: TimeInterval, target: String?) async throws -> String { + guard capturedAt == lastTimestamp, let target else { return try await readSymlink(path: path) } + return target + } + /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { let node = try node(at: path, followFinalSymlink: false) diff --git a/Sources/Workspace/RevisionComparison.swift b/Sources/Workspace/RevisionComparison.swift new file mode 100644 index 0000000..def7b2f --- /dev/null +++ b/Sources/Workspace/RevisionComparison.swift @@ -0,0 +1,192 @@ +import Foundation + +/// A checkpoint plus one live metadata walk. Only suspect files acquire bytes; +/// unchanged nodes continue to refer to the immutable checkpoint. +struct RevisionComparison: Sendable { + var checkpoint: TransactionTree + var current: TransactionTree + var metadata: MetadataIndex + var changed: Set + + static func capture( + _ revision: CapturedRevision, from fs: any FileSystem, + policy: Workspace.CheckpointCapturePolicy, load: @escaping ChangeSet.RevisionLoader + ) async throws -> Self { + let checkpoint = TransactionTree(index: revision.index, load: load) + var metadata = MetadataIndex() + metadata.exactTimestamps = await fs.supportsExactContentTimestamps + var nodes: [WorkspacePath: Workspace.MergeNode] = [:] + var changed: Set = [] + try await fs.walk { info, read, readLink in + let target = info.kind == .symlink ? try await readLink() : nil + let entry = MetadataIndex.Entry(info: info, target: target) + metadata.entries[info.path] = entry + var node = Workspace.MergeNode(kind: info.kind, permissions: info.permissions, symlinkTarget: target) + if info.kind == .file { + if policy == .trustMetadata, let old = revision.metadata.entries[info.path], + entry.canReuseContents(old, capturedAt: revision.metadata.capturedAt, + exactTimestamps: metadata.exactTimestamps && revision.metadata.exactTimestamps), + let existing = checkpoint.nodes[info.path], existing.kind == .file, + existing.permissions == info.permissions { + node = existing + node.permissions = info.permissions + } else { + let bytes = try await read(nil) + node = .init(kind: .file, permissions: info.permissions, fileData: bytes, + contentID: SHA256.hexDigest(of: bytes), size: UInt64(bytes.count)) + } + } + if node != checkpoint.nodes[info.path] { changed.insert(info.path) } + nodes[info.path] = node + } + changed.formUnion(checkpoint.nodes.keys.filter { nodes[$0] == nil }) + return Self(checkpoint: checkpoint, current: TransactionTree(nodes: nodes, load: load), + metadata: metadata, changed: changed) + } + + /// Matching directories are sparse containers, including permission-only + /// changes. Structural replacements own their complete old/new subtrees. + func restoreScope() -> Set { + var directories: Set = [] + for path in changed { + var ancestor = path + while true { + if current.nodes[ancestor]?.kind == .directory, + checkpoint.nodes[ancestor]?.kind == .directory { directories.insert(ancestor) } + if ancestor.isRoot { break } + ancestor = ancestor.parent + } + } + // A no-op still publishes a checkpoint, with an empty root patch. + if current.nodes[.root]?.kind == .directory, checkpoint.nodes[.root]?.kind == .directory { + directories.insert(.root) + } + return directories + } + + func sparseSnapshot(in tree: TransactionTree, preserving directories: Set) async throws -> Snapshot { + var children: [WorkspacePath: Set] = [:] + for path in changed.union(directories) where !path.isRoot { + children[path.parent, default: []].insert(path) + } + func entry(_ path: WorkspacePath) async throws -> Snapshot.Entry { + if directories.contains(path), let node = tree.nodes[path] { + var entries: [Snapshot.Entry] = [] + for child in (children[path] ?? []).sorted() { entries.append(try await entry(child)) } + return .directory(.init(path: path, permissions: node.permissions, children: entries)) + } + return try await tree.snapshot(at: path).entry + } + return Snapshot(rootPath: .root, entry: try await entry(.root)) + } +} + +extension TransactionTree { + func revisionIndex(paths: Set? = nil) -> RevisionIndex { + var selectedChildren = children + if let paths { + var selected = paths + for path in paths { + var ancestor = path + while !ancestor.isRoot { + ancestor = ancestor.parent + selected.insert(ancestor) + } + } + selectedChildren = [:] + for path in selected where !path.isRoot { selectedChildren[path.parent, default: []].append(path) } + } + func entry(_ path: WorkspacePath) -> RevisionIndex.Entry { + guard let node = nodes[path] else { return .missing(path: path) } + switch node.kind { + case .file: + return .file(path: path, size: node.size, permissions: node.permissions, contentID: node.contentID!) + case .symlink: + return .symbolicLink(path: path, target: node.symlinkTarget!, permissions: node.permissions) + case .directory: + return .directory(path: path, permissions: node.permissions, + children: (selectedChildren[path] ?? []).sorted().map(entry)) + } + } + return RevisionIndex(id: UUID(), root: .root, entry: entry(.root)) + } +} + +extension Snapshot { + /// Recapture only the scope actually restored. Ancestor modes are observed + /// without traversing their unaffected children. + static func captureSparse( + from fs: any FileSystem, template: Snapshot, preserving directories: Set, + metadata: inout MetadataIndex, previousChildren: [WorkspacePath: [WorkspacePath]] + ) async throws -> Snapshot { + func forget(_ path: WorkspacePath) { + metadata.entries.removeValue(forKey: path) + for child in previousChildren[path] ?? [] { forget(child) } + } + func capture(_ template: Entry) async throws -> Entry { + let path = template.path + if directories.contains(path), case let .directory(directory) = template { + let info = try await fs.stat(path: path) + metadata.entries[path] = .init(info: info, target: nil) + var children: [Entry] = [] + for child in directory.children { children.append(try await capture(child)) } + return .directory(.init(path: path, permissions: info.permissions, children: children)) + } + forget(path) + var children: [WorkspacePath: [Entry]] = [:] + var root: Entry = .missing(.init(path: path)) + try await fs.walk(at: path) { info, read, readLink in + let target = info.kind == .symlink ? try await readLink() : nil + metadata.entries[info.path] = .init(info: info, target: target) + let entry: Entry + switch info.kind { + case .file: + let bytes = try await read(nil) + entry = .file(.init(path: info.path, data: bytes, permissions: info.permissions, + contentID: SHA256.hexDigest(of: bytes))) + case .symlink: + entry = .symlink(.init(path: info.path, target: target!, permissions: info.permissions)) + case .directory: + entry = .directory(.init(path: info.path, permissions: info.permissions, + children: (children.removeValue(forKey: info.path) ?? []).sorted { $0.path < $1.path })) + } + if info.path == path { root = entry } + else { children[info.path.parent, default: []].append(entry) } + } + return root + } + return Snapshot(rootPath: template.rootPath, entry: try await capture(template.entry)) + } +} + +extension FilesystemObservation { + /// Retain cached bytes outside the restored scope. Replacing the observation + /// root with a sparse snapshot would discard those bytes and turn later + /// unrelated external edits into cache misses. + mutating func reconcileRestoration( + _ entry: Snapshot.Entry, preserving directories: Set, metadata replacement: MetadataIndex + ) { + if directories.contains(entry.path), case let .directory(directory) = entry, + metadata.entries[entry.path]?.info.kind == .directory { + metadata.entries[entry.path] = replacement.entries[entry.path] + for child in directory.children { + reconcileRestoration(child, preserving: directories, metadata: replacement) + } + return + } + var scoped = MetadataIndex() + if directories.contains(entry.path) { + // Observation may predate an out-of-band structural change. Rebuild + // that subtree's metadata from the walk already completed by restore. + scoped.entries = replacement.entries.filter { $0.key.isWithin(entry.path) } + } else { + func collect(_ entry: Snapshot.Entry) { + scoped.entries[entry.path] = replacement.entries[entry.path] + if case let .directory(directory) = entry { directory.children.forEach(collect) } + } + collect(entry) + } + replace(root: entry.path, metadata: scoped) + remember(entry) + } +} diff --git a/Sources/Workspace/Snapshot.swift b/Sources/Workspace/Snapshot.swift index feb2a11..93cdd81 100644 --- a/Sources/Workspace/Snapshot.swift +++ b/Sources/Workspace/Snapshot.swift @@ -129,15 +129,28 @@ extension Snapshot { at path: WorkspacePath = .root, snapshotId: UUID = UUID() ) async throws -> Snapshot { - Snapshot( - id: snapshotId, - rootPath: path, - entry: try await captureEntry(from: target, at: path) - ) + var metadata: MetadataIndex? + return try await capture(from: target, at: path, snapshotId: snapshotId, metadata: &metadata) } - static func restore(_ snapshot: Snapshot, to target: any FileSystem) async throws { - try await restoreEntry(snapshot.entry, to: target) + /// Pass a non-nil index to collect capture hints from the same entries and reads. + static func capture( + from target: any FileSystem, + at path: WorkspacePath = .root, + snapshotId: UUID = UUID(), + metadata: inout MetadataIndex? + ) async throws -> Snapshot { + if metadata != nil { + metadata = MetadataIndex() + metadata?.exactTimestamps = await target.supportsExactContentTimestamps + } + let entry = try await captureEntry(from: target, at: path, metadata: &metadata) + return Snapshot(id: snapshotId, rootPath: path, entry: entry) + } + + static func restore(_ snapshot: Snapshot, to target: any FileSystem, + preservingDirectories: Set = []) async throws { + try await restoreEntry(snapshot.entry, to: target, preservingDirectories: preservingDirectories) } static func infoIfPresent(at path: WorkspacePath, on target: any FileSystem) async throws -> FileInfo? { @@ -188,10 +201,14 @@ extension Snapshot { return false } - private static func captureTree(from target: any FileSystem, at path: WorkspacePath) async throws -> Entry { + private static func captureTree( + from target: any FileSystem, at path: WorkspacePath, metadata: inout MetadataIndex? + ) async throws -> Entry { var children: [WorkspacePath: [Entry]] = [:] var root: Entry = .missing(.init(path: path)) try await target.walk(at: path) { info, read, readLink in + let link = info.kind == .symlink ? try await readLink() : nil + metadata?.entries[info.path] = .init(info: info, target: link) let entry: Entry switch info.kind { case .directory: @@ -200,7 +217,7 @@ extension Snapshot { case .file: entry = .file(.init(path: info.path, data: try await read(nil), permissions: info.permissions)) case .symlink: - entry = .symlink(.init(path: info.path, target: try await readLink(), permissions: info.permissions)) + entry = .symlink(.init(path: info.path, target: link!, permissions: info.permissions)) } if info.path == path { root = entry } else { children[info.path.parent, default: []].append(entry) } @@ -208,25 +225,33 @@ extension Snapshot { return root } - private static func captureEntry(from target: any FileSystem, at path: WorkspacePath) async throws -> Entry { + private static func captureEntry( + from target: any FileSystem, at path: WorkspacePath, metadata: inout MetadataIndex? + ) async throws -> Entry { + // Listings do not authorize stat access to each child. Preserve both the + // per-entry check and the missing-child behavior of the per-path route. guard let info = try await infoIfPresent(at: path, on: target) else { return .missing(.init(path: path)) } + if info.kind == .directory && target.supportsScopedWalk { + return try await captureTree(from: target, at: path, metadata: &metadata) + } + let link = info.kind == .symlink ? try await target.readSymlink(path: path) : nil + metadata?.entries[path] = .init(info: info, target: link) switch info.kind { case .directory: - if target.supportsScopedWalk { return try await captureTree(from: target, at: path) } let entries = try await target.listDirectory(path: path) - let children = try await entries - .sorted { PathName($0.name) < PathName($1.name) } - .snapshotAsyncMap { entry in - try await captureEntry(from: target, at: path.appending(entry.name)) - } + var children: [Entry] = [] + children.reserveCapacity(entries.count) + for entry in entries.sorted(by: { PathName($0.name) < PathName($1.name) }) { + children.append(try await captureEntry(from: target, at: path.appending(entry.name), metadata: &metadata)) + } return .directory(.init(path: path, permissions: info.permissions, children: children)) case .symlink: return .symlink( .init( path: path, - target: try await target.readSymlink(path: path), + target: link!, permissions: info.permissions ) ) @@ -241,7 +266,8 @@ extension Snapshot { } } - private static func restoreEntry(_ entry: Entry, to target: any FileSystem) async throws { + private static func restoreEntry(_ entry: Entry, to target: any FileSystem, + preservingDirectories: Set) async throws { switch entry { case let .missing(missing): try await removeEntryIfPresent(at: missing.path, on: target) @@ -282,7 +308,8 @@ extension Snapshot { case let .directory(directory): try await ensureDirectory(at: directory.path, on: target) try await syncDirectoryChildren( - directory.children, under: directory.path, finalPermissions: directory.permissions, on: target + directory.children, under: directory.path, finalPermissions: directory.permissions, on: target, + preservingDirectories: preservingDirectories ) } } @@ -330,10 +357,11 @@ extension Snapshot { _ children: [Entry], under path: WorkspacePath, finalPermissions: POSIXPermissions? = nil, - on target: any FileSystem + on target: any FileSystem, + preservingDirectories: Set = [] ) async throws { do { - try await syncChildren(children, under: path, on: target) + try await syncChildren(children, under: path, on: target, preservingDirectories: preservingDirectories) } catch let deniedError { guard isPermissionDenied(deniedError) else { throw deniedError } let original = try await target.stat(path: path).permissions @@ -347,7 +375,7 @@ extension Snapshot { } do { // Synchronization is idempotent even if the first attempt made progress. - try await syncChildren(children, under: path, on: target) + try await syncChildren(children, under: path, on: target, preservingDirectories: preservingDirectories) try await setPermissionsIfNeeded(finalPermissions ?? original, at: path, on: target) } catch { do { @@ -367,17 +395,19 @@ extension Snapshot { private static func syncChildren( _ expectedChildren: [Entry], under parentPath: WorkspacePath, - on target: any FileSystem + on target: any FileSystem, + preservingDirectories: Set ) async throws { - let expectedNames = Set(expectedChildren.map { PathName($0.path.basename) }) - let existingEntries = try await target.listDirectory(path: parentPath) - - for entry in existingEntries where !expectedNames.contains(PathName(entry.name)) { - try await removeRecursively(at: parentPath.appending(entry.name), on: target) + if !preservingDirectories.contains(parentPath) { + let expectedNames = Set(expectedChildren.map { PathName($0.path.basename) }) + let existingEntries = try await target.listDirectory(path: parentPath) + for entry in existingEntries where !expectedNames.contains(PathName(entry.name)) { + try await removeRecursively(at: parentPath.appending(entry.name), on: target) + } } for child in expectedChildren.sorted(by: { $0.path < $1.path }) { - try await restoreEntry(child, to: target) + try await restoreEntry(child, to: target, preservingDirectories: preservingDirectories) } } @@ -404,19 +434,6 @@ extension Snapshot { } } -private extension Array { - func snapshotAsyncMap( - _ transform: @escaping @Sendable (Element) async throws -> T - ) async throws -> [T] { - var values: [T] = [] - values.reserveCapacity(count) - for element in self { - values.append(try await transform(element)) - } - return values - } -} - // A journal holds disjoint subtrees, preserving the state before the first edit // affecting each path. Expanding a root folds previously captured children back // into the newly captured ancestor, rather than recording intermediate edits. diff --git a/Sources/Workspace/Workspace+Checkpoints.swift b/Sources/Workspace/Workspace+Checkpoints.swift index 64a9fe0..bef33c9 100644 --- a/Sources/Workspace/Workspace+Checkpoints.swift +++ b/Sources/Workspace/Workspace+Checkpoints.swift @@ -38,28 +38,54 @@ extension Workspace { try await prepareForMutation() try await reconcileCheckpointsWithStore() let checkpoint = try checkpointOrThrow(id: checkpointId) - let targetSnapshot = try await loadSnapshotOrThrow( - id: checkpoint.snapshotId, - workspaceId: checkpoint.workspaceId - ) - let previousSnapshot = try await Snapshot.capture(from: filesystem) + let (comparison, pin) = try await checkpointComparison(checkpoint) + defer { withExtendedLifetime(pin) {} } + let directories = comparison.restoreScope() + let before = try await comparison.sparseSnapshot(in: comparison.current, preserving: directories) + let target = try await comparison.sparseSnapshot(in: comparison.checkpoint, preserving: directories) do { - try await Snapshot.restore(targetSnapshot, to: filesystem) - let restoredSnapshot = try await Snapshot.capture(from: filesystem) - let changes = changeSet(from: previousSnapshot, to: restoredSnapshot) - let result = try await persistMutationCheckpoint( - snapshot: restoredSnapshot, - label: label, - origin: .rollback(from: checkpoint.id), - operation: .rollback, - changes: recordingChanges(changes, from: previousSnapshot, to: restoredSnapshot) - ) - await noteWorkspaceSnapshot(restoredSnapshot) + try await Snapshot.restore(target, to: filesystem, preservingDirectories: directories) + var metadata = comparison.metadata + let after = try await Snapshot.captureSparse(from: filesystem, template: target, + preserving: directories, metadata: &metadata, + previousChildren: comparison.current.children) + let changes = changeSet(from: before, to: after) + let recorded = recordingChanges(changes, from: before, to: after) + let result = try await withMutationCheckpoint(operation: .rollback, changes: recorded) { + try await store.publishRevision( + reusing: checkpoint.snapshotId, patch: after, preservingDirectories: directories, + metadata: metadata, draft: CheckpointDraft(workspaceID: workspaceId, label: label, + preferredParentID: headCheckpointId, origin: .rollback(from: checkpoint.id))) + } + checkpoints.append(result) + checkpoints.sort(by: checkpointSort) + headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) + if var observed = observedFilesystem { + observed.reconcileRestoration(after.entry, preserving: directories, metadata: metadata) + observedFilesystem = observed + } + emitWorkspaceEvent(.checkpoint(result)) if !changes.isEmpty { emitWorkspaceEvent(.changes(changes)) } return result } catch { - try await restoreAfterFailure([previousSnapshot], originalError: error) + do { try await Snapshot.restore(before, to: filesystem, preservingDirectories: directories) } + catch let rollbackError { throw WorkspaceRollbackError(originalError: error, rollbackError: rollbackError) } + throw error + } + } + + func checkpointComparison(_ checkpoint: Checkpoint) async throws -> (RevisionComparison, TransactionRevisionPin) { + let store = store + let (revision, pin) = try await TransactionRevisionGate.shared.withLock { + guard let revision = try await store.loadCapturedRevision(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) else { + throw WorkspaceError.revisionDataNotFound(checkpoint.snapshotId) + } + return (revision, TransactionRevisionPin(checkpoint)) } + let content = TransactionRevisionContent(checkpoint: checkpoint, store: store) + let comparison = try await RevisionComparison.capture(revision, from: filesystem, + policy: checkpointCapture, load: { try await content.read($0) }) + return (comparison, pin) } /// Returns one checkpoint owned by this workspace when present. diff --git a/Sources/Workspace/Workspace+Internals.swift b/Sources/Workspace/Workspace+Internals.swift index fe991f3..dfe5031 100644 --- a/Sources/Workspace/Workspace+Internals.swift +++ b/Sources/Workspace/Workspace+Internals.swift @@ -125,17 +125,26 @@ extension Workspace { return persisted } - // If checkpoint persistence fails after the history append, retract exactly - // our record. Other instances' mutations must not be removed by compensation. func persistMutationCheckpoint( snapshot: Snapshot, label: String?, origin: Checkpoint.Origin, operation: Mutation.Operation, changes: ChangeSet ) async throws -> Checkpoint { - let mutation = changes.isEmpty ? nil : try await appendMutation(operation: operation, changes: changes) - do { - return try await persistCheckpoint( + try await withMutationCheckpoint(operation: operation, changes: changes) { + try await persistCheckpoint( snapshot: snapshot, label: label, parentCheckpointId: headCheckpointId, origin: origin ) + } + } + + // If checkpoint persistence fails after the history append, retract exactly + // our record. Other instances' mutations must not be removed by compensation. + func withMutationCheckpoint( + operation: Mutation.Operation, changes: ChangeSet, + publish: () async throws -> Checkpoint + ) async throws -> Checkpoint { + let mutation = changes.isEmpty ? nil : try await appendMutation(operation: operation, changes: changes) + do { + return try await publish() } catch { if let mutation { do { diff --git a/Sources/Workspace/Workspace+TransactionStorage.swift b/Sources/Workspace/Workspace+TransactionStorage.swift index 5a8d074..c054b93 100644 --- a/Sources/Workspace/Workspace+TransactionStorage.swift +++ b/Sources/Workspace/Workspace+TransactionStorage.swift @@ -60,8 +60,7 @@ extension Workspace { func persistMutationCheckpoint(tree: TransactionTree, metadata previousMetadata: MetadataIndex, changedRoots: [WorkspacePath], label: String?, origin: Checkpoint.Origin, operation: Mutation.Operation, changes: ChangeSet) async throws -> Checkpoint { - let mutation = changes.isEmpty ? nil : try await appendMutation(operation: operation, changes: changes) - do { + try await withMutationCheckpoint(operation: operation, changes: changes) { var metadata = previousMetadata for root in changedRoots { let replacement = try await MetadataIndex.capture(from: filesystem, at: root) @@ -88,16 +87,6 @@ extension Workspace { coldTransactionBase = nil emitWorkspaceEvent(.checkpoint(checkpoint)) return checkpoint - } catch { - if let mutation { - do { - try await store.removeMutation(sequence: mutation.sequence, workspaceId: workspaceId) - mutations.removeAll { $0.sequence == mutation.sequence } - } catch let cleanupError { - throw WorkspaceRollbackError(originalError: error, rollbackError: cleanupError) - } - } - throw error } } } diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 5e96d00..04fce86 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -8,11 +8,11 @@ public actor Workspace { case directory(URL) } - /// Controls content verification for transactions and directory checkpoint capture. + /// Controls content verification for transactions, checkpoint comparisons/restores, and directory checkpoint capture. public enum CheckpointCapturePolicy: Sendable, Equatable { /// Reuse matching size and mtime when mtime is over two seconds older than the prior /// capture, comparing identity when both entries provide it. Exact-timestamp backends - /// skip the transaction racy window. Missing mtimes always require reading contents. + /// skip the racy window. Missing mtimes always require reading contents. case trustMetadata /// Read and verify every file, including changes that preserve older timestamps. case verifyContents diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index 9c27af9..8a55c2c 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -26,13 +26,27 @@ extension Workspace { if let maxTextBytes = options.maxTextBytes, maxTextBytes < 0 { throw WorkspaceError.unsupported("text diff byte limit must not be negative") } - let currentSnapshot: Snapshot? = if source == .current || destination == .current { - try await Snapshot.capture(from: filesystem) - } else { - nil + if source == .current && destination == .current { + return ChangeSet(changes: []) + } + if source == .current || destination == .current { + await acquireOperation() + defer { releaseOperation() } + let checkpointID: UUID + if case let .checkpoint(id) = source { checkpointID = id } + else if case let .checkpoint(id) = destination { checkpointID = id } + else { preconditionFailure("one revision must be a checkpoint") } + try await reconcileCheckpointsWithStore() + let (comparison, pin) = try await checkpointComparison(checkpointOrThrow(id: checkpointID)) + defer { withExtendedLifetime(pin) {} } + let before = source == .current ? comparison.current : comparison.checkpoint + let after = destination == .current ? comparison.current : comparison.checkpoint + return try await ChangeSet.compare(before: before.revisionIndex(paths: comparison.changed), after: after.revisionIndex(paths: comparison.changed), + maxTextBytes: options.maxTextBytes, loadBefore: { try await before.data($0) }, + loadAfter: { try await after.data($0) }) } - async let before = resolvedRevision(source, currentSnapshot: currentSnapshot) - async let after = resolvedRevision(destination, currentSnapshot: currentSnapshot) + async let before = resolvedRevision(source) + async let after = resolvedRevision(destination) let (beforeRevision, afterRevision) = try await (before, after) return try await ChangeSet.compare( before: beforeRevision.index, diff --git a/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift b/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift index b947465..d8a9304 100644 --- a/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift +++ b/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift @@ -8,12 +8,12 @@ import Glibc #endif struct CheckpointCaptureTraversalTests { - @Test(arguments: [Workspace.CheckpointCapturePolicy.trustMetadata, .verifyContents]) - func captureRoutesPersistIdenticalTrees(policy: Workspace.CheckpointCapturePolicy) async throws { + @Test(arguments: [Workspace.CheckpointCapturePolicy.trustMetadata, .verifyContents], [false, true]) + func captureRoutesPersistIdenticalTrees(policy: Workspace.CheckpointCapturePolicy, inMemory: Bool) async throws { let container = try TestSupport.temporaryDirectory("CaptureRoutes") defer { TestSupport.remove(container) } let files = container.appendingPathComponent("files") - let local = try LocalFileSystem(root: files) + let local: any FileSystem = inMemory ? InMemoryFileSystem() : try LocalFileSystem(root: files) try await local.createDirectory(path: "/a/empty", recursive: true) try await local.createFile(path: "/a/file", data: Data([0, 1, 255])) try await local.createFile(path: "/a-", data: Data("ordering\n".utf8)) @@ -22,23 +22,29 @@ struct CheckpointCaptureTraversalTests { try await local.createHardLink(path: "/hard", target: "/a/file") for (name, target) in [("dangling", "missing"), ("cycle", "cycle"), ("alias", "a"), ("escape", container.appendingPathComponent("outside").path)] { - try FileManager.default.createSymbolicLink(atPath: files.appendingPathComponent(name).path, - withDestinationPath: target) + if inMemory { try await local.createSymlink(path: WorkspacePath("/" + name), target: target) } + else { + try FileManager.default.createSymbolicLink(atPath: files.appendingPathComponent(name).path, + withDestinationPath: target) + } } try Data("outside".utf8).write(to: container.appendingPathComponent("outside")) // macOS volumes cannot host both spellings. A real mounted backend can, // and also exercises forwarding plus mixed bulk/per-path traversal. - let unicode = InMemoryFileSystem() - try await unicode.createFile(path: "/caf\u{e9}", data: Data([1])) - try await unicode.createFile(path: "/cafe\u{301}", data: Data([2])) - let source = MountedFileSystem(base: local, mounts: [.init(mountPoint: "/unicode", fileSystem: unicode)]) + let unicode: any FileSystem = inMemory ? local : InMemoryFileSystem() + let unicodeRoot: WorkspacePath = inMemory ? "/unicode" : .root + if inMemory { try await unicode.createDirectory(path: unicodeRoot, recursive: true) } + try await unicode.createFile(path: unicodeRoot.appending("caf\u{e9}"), data: Data([1])) + try await unicode.createFile(path: unicodeRoot.appending("cafe\u{301}"), data: Data([2])) + let source: any FileSystem = inMemory ? local : + MountedFileSystem(base: local, mounts: [.init(mountPoint: "/unicode", fileSystem: unicode)]) let bulk = CaptureSource(source, scoped: true) let paths = CaptureSource(source, scoped: false) let id = UUID() let roots = [container.appendingPathComponent("bulk"), container.appendingPathComponent("paths")] let stores = roots.map { FileCheckpointStore(rootDirectory: $0, checkpointCapture: policy) } let old = Date(timeIntervalSince1970: 1) - for name in ["a/file", "a-"] { + for name in inMemory ? [] : ["a/file", "a-"] { try FileManager.default.setAttributes([.modificationDate: old], ofItemAtPath: files.appendingPathComponent(name).path) } var parents: [UUID?] = [nil, nil] @@ -47,7 +53,7 @@ struct CheckpointCaptureTraversalTests { if phase == 2 { try await local.createFile(path: "/a/new", data: Data("new\n".utf8)) try await local.setPermissions(path: "/a/file", permissions: POSIXPermissions(0o600)) - try await unicode.writeFile(path: "/cafe\u{301}", data: Data([3]), append: false) + try await source.writeFile(path: "/unicode/cafe\u{301}", data: Data([3]), append: false) } if phase == 3 { try await local.remove(path: "/a/new", recursive: false) @@ -63,6 +69,20 @@ struct CheckpointCaptureTraversalTests { workspaceID: id, label: nil, preferredParentID: parents[i], origin: .manual)) parents[i] = checkpoint.id summaries.append(checkpoint.summary) + let captured = try #require(await stores[i].loadCapturedRevision(id: checkpoint.snapshotId, workspaceId: id)) + let expected = try await MetadataIndex.capture(from: source) + #expect(Set(captured.metadata.entries.keys) == Set(expected.entries.keys)) + for (path, entry) in expected.entries { + let hint = try #require(captured.metadata.entries[path]) + #expect(hint.info.identity == entry.info.identity) + #expect(hint.info.modificationDate == entry.info.modificationDate) + #expect(hint.info.permissions == entry.info.permissions) + #expect(hint.info.kind == entry.info.kind) + #expect(hint.target == entry.target) + if entry.info.kind == .file { #expect(hint.info.size == entry.info.size) } + } + #expect(captured.metadata.exactTimestamps == expected.exactTimestamps) + #expect(captured.metadata.exactTimestamps == inMemory) let url = roots[i].appendingPathComponent(id.uuidString) .appendingPathComponent("snapshots/\(checkpoint.snapshotId).json") var manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: url)) as? [String: Any]) @@ -81,7 +101,7 @@ struct CheckpointCaptureTraversalTests { #expect(bulk.walkCount == 1) #expect(bulk.statCount == 0 && bulk.listCount == 0) #expect(bulk.reads == paths.reads) - if phase == 1, policy == .trustMetadata { + if phase == 1, policy == .trustMetadata, !inMemory { #expect(!bulk.reads.contains("/a/file")) #expect(!bulk.reads.contains("/hard")) #expect(!bulk.reads.contains("/a-")) @@ -97,7 +117,7 @@ struct CheckpointCaptureTraversalTests { let files = container.appendingPathComponent("files") let local = try LocalFileSystem(root: files) let source = CaptureSource(local, scoped: scoped) - let dates = ["stable": Date(timeIntervalSince1970: 1), "racy": Date().addingTimeInterval(60)] + let dates = ["stable": Date(timeIntervalSince1970: 1), "replaced": Date(timeIntervalSince1970: 1), "racy": Date().addingTimeInterval(60)] for (name, date) in dates { try await local.createFile(path: WorkspacePath(normalizing: "/" + name), data: Data("old".utf8)) try FileManager.default.setAttributes([.modificationDate: date], ofItemAtPath: files.appendingPathComponent(name).path) @@ -108,14 +128,22 @@ struct CheckpointCaptureTraversalTests { let first = try await store.captureRevision(from: source, draft: .init( workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) for (name, date) in dates { - try await local.writeFile(path: WorkspacePath(normalizing: "/" + name), data: Data("new".utf8), append: false) + if name == "replaced" { + // Atomic replacement changes identity despite equal size and old mtime. + try await local.writeFile(path: WorkspacePath(normalizing: "/" + name), data: Data("new".utf8), append: false) + } else { + // Preserve identity to isolate the timestamp-window contract. + let handle = try FileHandle(forWritingTo: files.appendingPathComponent(name)) + try handle.write(contentsOf: Data("new".utf8)) + try handle.close() + } try FileManager.default.setAttributes([.modificationDate: date], ofItemAtPath: files.appendingPathComponent(name).path) } source.reset() let second = try await store.captureRevision(from: source, draft: .init( workspaceID: id, label: nil, preferredParentID: first.id, origin: .manual)) - #expect(source.reads == ["/racy"]) - #expect(second.summary.touchedPaths == ["/racy"]) + #expect(source.reads == ["/racy", "/replaced"]) + #expect(second.summary.touchedPaths == ["/racy", "/replaced"]) #expect(try await store.readSnapshotFile(id: second.snapshotId, workspaceId: id, path: "/racy", offset: 0, length: nil) == Data("new".utf8)) // Repair an absent CAS object with the same bytes. A different payload // would still fail when the summary tries to read the damaged parent. @@ -130,6 +158,40 @@ struct CheckpointCaptureTraversalTests { #expect(try await store.readSnapshotFile(id: repaired.snapshotId, workspaceId: id, path: "/stable", offset: 0, length: nil) == Data("old".utf8)) } + @Test(arguments: [false, true]) + func identityOnlyReplacementRefreshesCaptureHints(scoped: Bool) async throws { + let container = try TestSupport.temporaryDirectory("CaptureIdentity") + defer { TestSupport.remove(container) } + let files = container.appendingPathComponent("files") + let local = try LocalFileSystem(root: files) + let source = CaptureSource(local, scoped: scoped) + let store = FileCheckpointStore(rootDirectory: container.appendingPathComponent("store")) + let id = UUID() + let old = Date(timeIntervalSince1970: 1) + var parent: UUID? + var previousIdentity: FileIdentity? + for phase in 0..<3 { + if phase < 2 { + try await local.writeFile(path: "/file", data: Data("same".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: old], + ofItemAtPath: files.appendingPathComponent("file").path) + } + let info = try await local.stat(path: "/file") + if phase == 1 { #expect(info.identity != previousIdentity) } + source.reset() + let checkpoint = try await store.captureRevision(from: source, draft: .init( + workspaceID: id, label: nil, preferredParentID: parent, origin: .manual)) + // An identity-only replacement must update the tree even though its + // content hash is unchanged, then allow the next capture to reuse it. + let captured = try #require(await store.loadCapturedRevision(id: checkpoint.snapshotId, workspaceId: id)) + #expect(captured.metadata.entries["/file"]?.info.identity == info.identity) + #expect(source.reads == (phase < 2 ? ["/file"] : [])) + if phase > 0 { #expect(checkpoint.summary.touchedPaths.isEmpty) } + previousIdentity = info.identity + parent = checkpoint.id + } + } + @Test(arguments: [Int32(ENOENT), Int32(ENOTDIR)]) func vanishedListedChildrenAreOmittedByBothRoutes(code: Int32) async throws { let root = try TestSupport.temporaryDirectory("CaptureVanished") @@ -157,11 +219,14 @@ struct CheckpointCaptureTraversalTests { #expect(trees[0] == trees[1]) } - @Test - func captureRoutesProduceIdenticalPackLayout() async throws { + @Test(arguments: [false, true]) + func captureRoutesProduceIdenticalPackLayout(inMemory: Bool) async throws { let container = try TestSupport.temporaryDirectory("CapturePacks") defer { TestSupport.remove(container) } - let local = try LocalFileSystem(root: container.appendingPathComponent("files")) + let local: any FileSystem = inMemory ? InMemoryFileSystem() : try LocalFileSystem(root: container.appendingPathComponent("files")) + for path: WorkspacePath in ["/a", "/a0"] { + try await local.createDirectory(path: path, recursive: true) + } for i in 0..<32 { try await local.createFile(path: WorkspacePath(normalizing: String(format: "/%02d", i)), data: Data([UInt8(i)])) } for name in ["a/leaf", "a-", "a0/leaf", "a0-", "z"] { try await local.createFile(path: WorkspacePath(normalizing: "/" + name), data: Data(name.utf8)) @@ -227,7 +292,8 @@ struct CheckpointCaptureTraversalTests { } /// Records reads on the actual descriptor callback as well as the per-path route. -private final class CaptureSource: FileSystem, TraversingFileSystem, @unchecked Sendable { +private final class CaptureSource: FileSystem, TraversingFileSystem, ExactTimestampFileSystem, @unchecked Sendable { + var hasExactContentTimestamps: Bool { get async { await base.supportsExactContentTimestamps } } let base: any FileSystem let supportsScopedWalk: Bool let failingRead: Bool diff --git a/Tests/WorkspaceTests/FileSystemTraversalTests.swift b/Tests/WorkspaceTests/FileSystemTraversalTests.swift index d9880da..4a4d19c 100644 --- a/Tests/WorkspaceTests/FileSystemTraversalTests.swift +++ b/Tests/WorkspaceTests/FileSystemTraversalTests.swift @@ -8,6 +8,74 @@ import Glibc #endif struct FileSystemTraversalTests { + @Test(arguments: [PermissionOperation.stat, .listDirectory, .readFile, .readSymlink], + ["snapshot", "checkpoint", "restore", "diff"]) + func revisionOperationsPreservePerEntryAuthorization(operation: PermissionOperation, action: String) async throws { + let root = try TestSupport.temporaryDirectory("RevisionAuthorization") + defer { TestSupport.remove(root) } + for local in [false, true] { + for wrapper in ["direct", "limited", "mounted", "overlay"] { + let base: any FileSystem = local ? try LocalFileSystem(root: root) : InMemoryFileSystem() + try await base.createDirectory(path: "/private", recursive: true) + try await base.writeFile(path: "/private/file", data: Data("secret".utf8), append: false) + if await base.exists(path: "/private/link") { try await base.remove(path: "/private/link", recursive: false) } + try await base.createSymlink(path: "/private/link", target: "file") + let denial = TraversalDenial() + let authorized = AuthorizedFileSystem(base: wrapper == "overlay" ? OverlayFileSystem(over: base) : base, authorizer: PermissionAuthorizer { request in + if await denial.enabled, request.operation == operation, + request.path?.string.hasPrefix("/private") == true { + return .deny(message: "per-entry denial") + } + return .allow + }) + let fs: any FileSystem + switch wrapper { + case "limited": fs = LimitedFileSystem(base: authorized, limits: .init()) + case "mounted": fs = MountedFileSystem(base: authorized, mounts: [ + .init(mountPoint: "/other", fileSystem: InMemoryFileSystem()) + ]) + default: fs = authorized + } + let workspace = Workspace(fileSystem: fs, checkpointCapture: .verifyContents) + let checkpoint = try await workspace.createCheckpoint() + try await base.writeFile(path: "/private/file", data: Data("changed".utf8), append: false) + let before = try await Snapshot.capture(from: base) + await denial.enable() + do { + switch action { + case "snapshot": _ = try await Snapshot.capture(from: fs) + case "checkpoint": _ = try await workspace.createCheckpoint() + case "restore": _ = try await workspace.restore(to: checkpoint) + default: _ = try await workspace.diff(from: .checkpoint(checkpoint.id), to: .current) + } + Issue.record("\(action) through \(wrapper) must enforce \(operation)") + } catch let WorkspaceError.accessDenied(actual, message) { + #expect(actual == operation.rawValue) + #expect(message == "per-entry denial") + } + #expect(try await Snapshot.capture(from: base).entry == before.entry) + } + } + } + + @Test + func perPathSnapshotRecordsVanishedChildrenAsMissing() async throws { + let base = InMemoryFileSystem() + try await base.writeFile(path: "/vanished", data: Data("old".utf8), append: false) + let fs = AuthorizedFileSystem(base: base, authorizer: PermissionAuthorizer { request in + if request.operation == .stat && request.path == "/vanished" { + do { try await base.remove(path: "/vanished", recursive: false) } + catch { Issue.record(error) } + } + return .allow + }) + let snapshot = try await Snapshot.capture(from: fs) + guard case let .directory(directory) = snapshot.entry else { + Issue.record("expected root directory"); return + } + #expect(directory.children == [.missing(.init(path: "/vanished"))]) + } + @Test func mountedWalkRebasesPathsAndPreservesShadowing() async throws { let root = try TestSupport.temporaryDirectory("MountedWalk") @@ -409,3 +477,8 @@ struct FileSystemTraversalTests { } private final class TraversalTestBundle: NSObject {} + +private actor TraversalDenial { + var enabled = false + func enable() { enabled = true } +} diff --git a/Tests/WorkspaceTests/InMemoryTraversalTests.swift b/Tests/WorkspaceTests/InMemoryTraversalTests.swift new file mode 100644 index 0000000..b785522 --- /dev/null +++ b/Tests/WorkspaceTests/InMemoryTraversalTests.swift @@ -0,0 +1,146 @@ +import Foundation +import Testing +@testable import Workspace + +struct InMemoryTraversalTests { + @Test(arguments: [false, true]) + func nativeWalkMatchesPerPathCaptureAndMetadata(skipUnreadable: Bool) async throws { + let fs = InMemoryFileSystem() + try await fs.createDirectory(path: "/a/empty", recursive: true) + for path: WorkspacePath in ["/a/file", "/a.txt", "/a0", "/é", "/e\u{301}"] { + try await fs.writeFile(path: path, data: Data([0, 1, 255]), append: false) + } + try await fs.setPermissions(path: "/a/file", permissions: POSIXPermissions(0o100)) + try await fs.setPermissions(path: "/a", permissions: POSIXPermissions(0o000)) + try await fs.createHardLink(path: "/hard", target: "/a/file") + for (name, target) in [("dangling", "missing"), ("cycle", "cycle"), ("alias", "a"), ("absolute", "/absent")] { + try await fs.createSymlink(path: WorkspacePath("/" + name), target: target) + } + let fallback = HookableFileSystem(base: fs) + for path: WorkspacePath in [.root, "/a", "/a/file", "/alias/empty", "/dangling", "/cycle", "/absolute", "/absent"] { + var nativeMetadata: MetadataIndex? = MetadataIndex() + var fallbackMetadata: MetadataIndex? = MetadataIndex() + let native = try await Snapshot.capture(from: fs, at: path, metadata: &nativeMetadata) + let reference = try await Snapshot.capture(from: fallback, at: path, metadata: &fallbackMetadata) + #expect(native.entry == reference.entry) + #expect(nativeMetadata?.entries == fallbackMetadata?.entries) + var nativeInfos: [FileInfo] = [] + var fallbackInfos: [FileInfo] = [] + var nativeBytes: [Data] = [] + var fallbackBytes: [Data] = [] + var nativeLinks: [String] = [] + var fallbackLinks: [String] = [] + for bulk in [true, false] { + let backend: any FileSystem = bulk ? fs : fallback + try await backend.walk(at: path, skippingUnreadableDirectories: skipUnreadable) { info, read, readLink in + if bulk { nativeInfos.append(info) } else { fallbackInfos.append(info) } + if info.kind == .file { + for length: Int? in [nil, 0, 2, 100] { + let bytes = try await read(length) + if bulk { nativeBytes.append(bytes) } else { fallbackBytes.append(bytes) } + } + await #expect(throws: WorkspaceError.self) { _ = try await read(-1) } + } else if info.kind == .symlink { + let target = try await readLink() + if bulk { nativeLinks.append(target) } else { fallbackLinks.append(target) } + } + } + } + #expect(nativeInfos.map(\.path) == fallbackInfos.map(\.path)) + #expect(nativeInfos.map { MetadataIndex.Entry(info: $0) } == fallbackInfos.map { MetadataIndex.Entry(info: $0) }) + #expect(nativeInfos.map(\.linkCount) == fallbackInfos.map(\.linkCount)) + #expect(nativeBytes == fallbackBytes) + #expect(nativeLinks == fallbackLinks) + if path.isRoot { + #expect(nativeInfos.filter { $0.kind == .file }.map(\.path) == ["/a.txt", "/a/file", "/a0", "/e\u{301}", "/hard", "/é"]) + #expect(nativeInfos.first { $0.path == "/hard" }?.linkCount == 2) + #expect(nativeInfos.first { $0.path == "/hard" }?.identity == nativeInfos.first { $0.path == "/a/file" }?.identity) + } + } + } + + @Test(arguments: [false, true], [ + "remove pending", "replace ancestor", "replace pending with link", "change bytes", + "remove during read", "change link target", "add hard link", "reset", "nested walk" + ]) + func reentrantWalkMatchesPerPathSemantics(skipVanished: Bool, mutation: String) async throws { + var results: [[WalkRecord]] = [] + var errors: [Int?] = [] + for route in ["native", "limited", "mounted", "fallback"] { + let fs = InMemoryFileSystem() + try await fs.createDirectory(path: "/dir", recursive: true) + for path: WorkspacePath in ["/a", "/dir/a", "/dir/z", "/target", "/z"] { + try await fs.writeFile(path: path, data: Data(path.string.utf8), append: false) + } + try await fs.createSymlink(path: "/alink", target: "a") + let backend: any FileSystem + switch route { + case "native": backend = fs + case "limited": backend = LimitedFileSystem(base: fs, limits: .init()) + case "mounted": backend = MountedFileSystem(base: fs) + default: backend = HookableFileSystem(base: fs) + } + var records: [WalkRecord] = [] + var errorCode: Int? + do { + try await backend.walk(skippingVanishedEntries: skipVanished) { info, read, readLink in + if info.path == "/a" { + switch mutation { + case "remove pending": try await fs.remove(path: "/z", recursive: false) + case "replace pending with link": + try await fs.remove(path: "/z", recursive: false) + try await fs.createSymlink(path: "/z", target: "target") + case "change bytes": try await fs.writeFile(path: "/a", data: Data("new".utf8), append: false) + case "remove during read": try await fs.remove(path: "/a", recursive: false) + case "add hard link": try await fs.createHardLink(path: "/new", target: "/z") + case "nested walk": + var paths: [WorkspacePath] = [] + try await fs.walk { nested, _, _ in paths.append(nested.path) } + #expect(paths.count == 8) + default: break + } + } + if info.path == "/alink" && mutation == "change link target" { + try await fs.remove(path: "/alink", recursive: false) + try await fs.createSymlink(path: "/alink", target: "target") + } + records.append(WalkRecord(path: info.path, kind: info.kind, permissions: info.permissions, + linkCount: info.linkCount, bytes: info.kind == .file ? try await read(nil) : nil, + target: info.kind == .symlink ? try await readLink() : nil)) + if info.path == "/dir/a" && mutation == "replace ancestor" { + try await fs.remove(path: "/dir", recursive: true) + try await fs.writeFile(path: "/dir", data: Data("replacement".utf8), append: false) + } + if info.path == "/a" && mutation == "reset" { + await fs.reset() + try await fs.writeFile(path: "/z", data: Data("replacement".utf8), append: false) + } + } + } catch { + let error = error as NSError + #expect(error.domain == NSPOSIXErrorDomain) + errorCode = error.code + } + // Skipping missing metadata never swallows a read failure after metadata. + let expectedError: Int? + if mutation == "remove during read" { expectedError = Int(ENOENT) } + else if mutation == "replace ancestor" && !skipVanished { expectedError = Int(ENOTDIR) } + else if ["remove pending", "reset"].contains(mutation) && !skipVanished { expectedError = Int(ENOENT) } + else { expectedError = nil } + #expect(errorCode == expectedError) + results.append(records) + errors.append(errorCode) + } + #expect(results.allSatisfy { $0 == results.last }) + #expect(errors.allSatisfy { $0 == errors.last! }) + } +} + +private struct WalkRecord: Equatable { + var path: WorkspacePath + var kind: FileTree.Kind + var permissions: POSIXPermissions + var linkCount: UInt64? + var bytes: Data? + var target: String? +} diff --git a/Tests/WorkspaceTests/RestoreDiffTests.swift b/Tests/WorkspaceTests/RestoreDiffTests.swift new file mode 100644 index 0000000..066b332 --- /dev/null +++ b/Tests/WorkspaceTests/RestoreDiffTests.swift @@ -0,0 +1,341 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Scoped checkpoint restore and diff", .serialized) +struct RestoreDiffTests { + @Test(arguments: [false, true]) + func `native walks collect checkpoint metadata without per child stats`(local: Bool) async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let base: any FileSystem = local ? try LocalFileSystem(root: root) : InMemoryFileSystem() + try await base.createDirectory(path: "/dir", recursive: true) + try await base.writeFile(path: "/dir/a", data: Data("old".utf8), append: false) + try await base.createSymlink(path: "/dangling", target: "../outside") + let expected = try await Snapshot.capture(from: base) + let expectedMetadata = try await MetadataIndex.capture(from: base) + let fs = HookableFileSystem(base: base, scopedWalk: true) + let store = InMemoryCheckpointStore() + let id = UUID() + let checkpoint = try await store.captureRevision(from: fs, draft: .init( + workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + #expect(await fs.statCount <= 1) + #expect(await fs.walks.count == 1) + let captured = try #require(try await store.loadCapturedRevision(id: checkpoint.snapshotId, workspaceId: id)) + #expect(captured.metadata.entries == expectedMetadata.entries) + #expect(captured.metadata.exactTimestamps == expectedMetadata.exactTimestamps) + #expect(try await store.loadSnapshot(id: checkpoint.snapshotId, workspaceId: id)?.entry == expected.entry) + } + + @Test + func `evicted capture metadata falls back to reading contents`() async throws { + let fs = CoarseFileSystem(mode: .coarse, timestamp: Date(timeIntervalSince1970: 1_000)) + try await fs.writeFile(path: "/a", data: Data("old".utf8), append: false) + let workspace = Workspace(fileSystem: fs) + var checkpoints: [Checkpoint] = [] + for _ in 0..<8 { checkpoints.append(try await workspace.createCheckpoint()) } + let store = await workspace.store + var retained = 0 + for checkpoint in checkpoints { + let revision = try #require(try await store.loadCapturedRevision( + id: checkpoint.snapshotId, workspaceId: workspace.workspaceID)) + if !revision.metadata.entries.isEmpty { retained += 1 } + } + #expect(retained > 0 && retained <= 4) + let oldest = checkpoints[0] + let evicted = try #require(try await store.loadCapturedRevision( + id: oldest.snapshotId, workspaceId: workspace.workspaceID)) + #expect(evicted.metadata.entries.isEmpty) + try await fs.writeFile(path: "/a", data: Data("new".utf8), append: false) + #expect(try await workspace.diff(from: .checkpoint(oldest.id), to: .current).touchedPaths == ["/a"]) + _ = try await workspace.restore(to: oldest) + #expect(try await fs.readFile(path: "/a") == Data("old".utf8)) + } + + @Test + func `current to current diff needs no filesystem access`() async throws { + let fs = HookableFileSystem() + let workspace = Workspace(fileSystem: fs) + await fs.failStat(NSError(domain: NSPOSIXErrorDomain, code: Int(EACCES))) + #expect(try await workspace.diff(from: .current, to: .current).isEmpty) + #expect(await fs.statCount == 0) + #expect(await fs.reads.isEmpty) + #expect(await fs.listings.isEmpty) + #expect(await fs.walks.isEmpty) + } + + // Extend the edit matrix with structural and byte-identity cases specific to + // restoring a prior tree. The reference always captures and restores whole trees. + static let scenarios = EditPathTests.scenarios + [ + .init(description: "file to directory", edits: [.remove("/dir/a"), .createDirectory("/dir/a/sub"), .writeText("/dir/a/sub/new", "new")]), + .init(description: "directory to file", edits: [.remove("/dir/sub"), .writeText("/dir/sub", "new")]), + .init(description: "directory to link", edits: [.remove("/dir/sub"), .createSymbolicLink("/dir/sub", target: "../outside")]), + .init(description: "link to directory", edits: [.remove("/dangling"), .createDirectory("/dangling/sub")]), + .init(description: "out of root link", edits: [.remove("/link"), .createSymbolicLink("/link", target: "../../outside")]), + .init(description: "byte distinct targets", edits: [.remove("/link"), .createSymbolicLink("/link", target: "cafe\u{301}")]), + .init(description: "byte distinct names", edits: [.writeText("/cafe\u{301}", "nfd"), .writeText("/Case", "upper")]), + .init(description: "binary mode only", edits: [.setPermissions("/binary", POSIXPermissions(0o600))]), + .init(description: "directory modes", edits: [.setPermissions("/dir/sub", POSIXPermissions(0o500)), .setPermissions("/dir", POSIXPermissions(0o500))]), + .init(description: "no change", edits: []), + ] + static let cases = [false, true].flatMap { local in + [false, true].flatMap { disk in + scenarios.filter { local || !$0.memoryRejects }.map { (local, disk, $0) } + } + } + + @Test(arguments: cases) + func `restore diff tree checkpoint and history equal whole tree reference`( + local: Bool, disk: Bool, scenario: EditPathTests.Scenario + ) async throws { + let root = try TestSupport.temporaryDirectory() + let referenceRoot = try TestSupport.temporaryDirectory() + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root); TestSupport.remove(referenceRoot); TestSupport.remove(storage) } + let fs: any FileSystem = local ? try LocalFileSystem(root: root) : InMemoryFileSystem() + let reference: any FileSystem = local ? try LocalFileSystem(root: referenceRoot) : InMemoryFileSystem() + try await fs.createDirectory(path: "/mnt/data", recursive: true) + try await fs.createDirectory(path: "/dir/sub", recursive: true) + try await fs.writeFile(path: "/dir/a", data: Data("alpha".utf8), append: false) + try await fs.writeFile(path: "/dir/sub/b", data: Data("beta".utf8), append: false) + try await fs.writeFile(path: "/binary", data: Data([0, 1, 2]), append: false) + for (path, target) in [("/dirlink", "dir"), ("/dir/link", "a"), ("/link", "dir/a"), + ("/dangling", "nowhere"), ("/chain", "dangling")] { + try await fs.createSymlink(path: WorkspacePath(path), target: target) + } + let persistence: Workspace.Persistence = disk ? .directory(storage) : .memory + var workspace = Workspace(fileSystem: fs, persistence: persistence) + let target = try await Snapshot.capture(from: fs) + let checkpoint = try await workspace.createCheckpoint() + if !scenario.edits.isEmpty { _ = try await workspace.apply(scenario.edits) } + let before = try await Snapshot.capture(from: fs) + let history = try await workspace.history() + if disk { + workspace = Workspace(workspaceID: workspace.workspaceID, fileSystem: fs, persistence: persistence) + } + let expectedDiff = try await ChangeSet.compare(before: RevisionIndex(snapshot: target), after: RevisionIndex(snapshot: before), + maxTextBytes: DiffOptions.default.maxTextBytes, + loadBefore: { path in try #require(Workspace.snapshotEntry(at: path, in: target.entry)?.fileBytes) }, + loadAfter: { path in try #require(Workspace.snapshotEntry(at: path, in: before.entry)?.fileBytes) }) + #expect(try await workspace.diff(from: .checkpoint(checkpoint.id), to: .current) == expectedDiff) + try await Snapshot.restore(before, to: reference) + try await Snapshot.restore(target, to: reference) + let expectedTree = try await Snapshot.capture(from: reference) + let expectedChanges = ChangeSet.compare(before: before, after: expectedTree, maxTextBytes: DiffOptions.default.maxTextBytes) + let restored = try await workspace.restore(to: checkpoint) + #expect(try await Snapshot.capture(from: fs).entry == expectedTree.entry) + let persisted = try await workspace.revisionSnapshot(.checkpoint(restored.id)) + #expect(persisted.entry == expectedTree.entry) + let actualHistory = try await workspace.history() + #expect(Array(actualHistory.prefix(history.count)) == history) + #expect(actualHistory.count == history.count + (expectedChanges.isEmpty ? 0 : 1)) + if !expectedChanges.isEmpty { + #expect(actualHistory.last?.changes == expectedChanges) + #expect(actualHistory.last?.operation == .rollback) + } + #expect(restored.origin == .rollback(from: checkpoint.id)) + #expect(restored.summary == ChangeSet.compare(before: target, after: expectedTree, + maxTextBytes: DiffOptions.default.maxTextBytes).summary) + } + + @Test(arguments: [false, true], [Workspace.RecordingPolicy.full(maxTextBytes: 0), .changesOnly, .off]) + func `restore respects recording policy independently of its changes`(disk: Bool, recording: Workspace.RecordingPolicy) async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + let fs = InMemoryFileSystem() + try await fs.writeFile(path: "/a", data: Data("before".utf8), append: false) + let workspace = Workspace(fileSystem: fs, persistence: disk ? .directory(storage) : .memory, recording: recording) + let checkpoint = try await workspace.createCheckpoint() + try await fs.writeFile(path: "/a", data: Data("after".utf8), append: false) + let before = try await Snapshot.capture(from: fs) + _ = try await workspace.restore(to: checkpoint) + let after = try await Snapshot.capture(from: fs) + let changes = ChangeSet.compare(before: before, after: after, maxTextBytes: DiffOptions.default.maxTextBytes) + let history = try await workspace.history() + if recording == .off { #expect(history.isEmpty) } + else if case let .full(limit) = recording { + #expect(history.last?.changes == ChangeSet.compare(before: before, after: after, maxTextBytes: limit)) + } else { + #expect(history.last?.changes.changes.map(\.path) == changes.changes.map(\.path)) + #expect(history.last?.changes.changes.allSatisfy { $0.diff == nil } == true) + } + } + + @Test(arguments: [false, true], [false, true]) + func `coarse metadata uses the capture window and verification policy after reopening`(disk: Bool, noIdentity: Bool) async throws { + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(storage) } + for recent in [false, true] { + for verify in [false, true] { + for changePermissions in [false, true] { + let fs = CoarseFileSystem(mode: noIdentity ? .coarse : .racy, + timestamp: recent ? Date().addingTimeInterval(60) : Date(timeIntervalSince1970: 1_000)) + try await fs.writeFile(path: "/a", data: Data("old".utf8), append: false) + let policy: Workspace.CheckpointCapturePolicy = verify ? .verifyContents : .trustMetadata + var workspace = Workspace(fileSystem: fs, persistence: disk ? .directory(storage) : .memory, checkpointCapture: policy) + let checkpoint = try await workspace.createCheckpoint() + if disk { + workspace = Workspace(workspaceID: workspace.workspaceID, fileSystem: fs, + persistence: .directory(storage), checkpointCapture: policy) + } + try await fs.writeFile(path: "/a", data: Data("new".utf8), append: false) + if changePermissions { try await fs.setPermissions(path: "/a", permissions: POSIXPermissions(0o600)) } + let before = try await Snapshot.capture(from: fs) + let expectedDetection = recent || verify || changePermissions + #expect(try await workspace.diff(from: .checkpoint(checkpoint.id), to: .current).isEmpty == !expectedDetection) + _ = try await workspace.restore(to: checkpoint) + #expect(try await fs.readFile(path: "/a") == Data((expectedDetection ? "old" : "new").utf8)) + let history = try await workspace.history() + #expect(history.isEmpty == !expectedDetection) + if expectedDetection { + let after = try await Snapshot.capture(from: fs) + #expect(history.last?.changes == ChangeSet.compare(before: before, after: after, + maxTextBytes: DiffOptions.default.maxTextBytes)) + } + } + } + } + } + + @Test(arguments: [false, true]) + func `identity replacement invalidates safely old matching metadata`(disk: Bool) async throws { + let root = try TestSupport.temporaryDirectory() + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root); TestSupport.remove(storage) } + let fs = try LocalFileSystem(root: root) + let timestamp = Date(timeIntervalSince1970: 1_000) + let url = root.appendingPathComponent("a") + try await fs.writeFile(path: "/a", data: Data("old".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: timestamp], ofItemAtPath: url.path) + var workspace = Workspace(fileSystem: fs, persistence: disk ? .directory(storage) : .memory) + let checkpoint = try await workspace.createCheckpoint() + let oldIdentity = try await fs.stat(path: "/a").identity + // Atomic replacement supplies a distinct inode, keeping the original + // alive at another name so immediate inode reuse cannot mask the test. + try await fs.move(from: "/a", to: "/old-inode") + try await fs.writeFile(path: "/a", data: Data("new".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: timestamp], ofItemAtPath: url.path) + #expect(try await fs.stat(path: "/a").identity != oldIdentity) + if disk { + workspace = Workspace(workspaceID: workspace.workspaceID, fileSystem: fs, persistence: .directory(storage)) + } + #expect(try await workspace.diff(from: .checkpoint(checkpoint.id), to: .current).touchedPaths.contains("/a")) + _ = try await workspace.restore(to: checkpoint) + #expect(try await fs.readFile(path: "/a") == Data("old".utf8)) + } + + @Test(arguments: [false, true]) + func `unchanged files need no content access for diff restore or publication`(disk: Bool) async throws { + let root = try TestSupport.temporaryDirectory() + let storage = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root); TestSupport.remove(storage) } + let base = try LocalFileSystem(root: root) + for name in ["changed", "untouched"] { + try await base.writeFile(path: WorkspacePath("/" + name), data: Data("old".utf8), append: false) + try FileManager.default.setAttributes([.modificationDate: Date(timeIntervalSince1970: 1_000)], + ofItemAtPath: root.appendingPathComponent(name).path) + } + let fs = HookableFileSystem(base: base, forwardsPermissions: true) + var workspace = Workspace(fileSystem: fs, persistence: disk ? .directory(storage) : .memory) + let checkpoint = try await workspace.createCheckpoint() + if disk { + workspace = Workspace(workspaceID: workspace.workspaceID, fileSystem: fs, persistence: .directory(storage)) + } + try await base.writeFile(path: "/changed", data: Data("new".utf8), append: false) + await fs.denyReads(at: "/untouched") + let readCount = await fs.reads.count + #expect(try await workspace.diff(from: .checkpoint(checkpoint.id), to: .current).touchedPaths == ["/changed"]) + _ = try await workspace.restore(to: checkpoint) + #expect(await fs.reads.dropFirst(readCount).allSatisfy { $0 == "/changed" }) + #expect(await fs.writes == ["/changed"]) + #expect(try await base.readFile(path: "/changed") == Data("old".utf8)) + } + + @Test(arguments: [false, true]) + func `sparse restore grants directory access only after denial`(local: Bool) async throws { + let root = try TestSupport.temporaryDirectory() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: root.appendingPathComponent("d").path) + TestSupport.remove(root) + } + let base: any FileSystem = local ? try LocalFileSystem(root: root) : InMemoryFileSystem() + try await base.createDirectory(path: "/d", recursive: true) + try await base.writeFile(path: "/d/a", data: Data("old".utf8), append: false) + try await base.setPermissions(path: "/d", permissions: POSIXPermissions(0o500)) + let fs = HookableFileSystem(base: base, forwardsPermissions: true) + let workspace = Workspace(fileSystem: fs) + let checkpoint = try await workspace.createCheckpoint() + try await base.setPermissions(path: "/d", permissions: POSIXPermissions(0o700)) + try await base.writeFile(path: "/d/a", data: Data("new".utf8), append: false) + try await base.setPermissions(path: "/d", permissions: POSIXPermissions(0o500)) + _ = try await workspace.restore(to: checkpoint) + #expect(try await base.readFile(path: "/d/a") == Data("old".utf8)) + #expect(try await base.stat(path: "/d").permissions == POSIXPermissions(0o500)) + if local && geteuid() != 0 { + #expect(await fs.permissionWrites.map(\.permissions) == [POSIXPermissions(0o700), POSIXPermissions(0o500)]) + } else { + #expect(await fs.permissionWrites.isEmpty) + } + } + + @Test + func `restore keeps cached text for unaffected observation paths`() async throws { + let fs = SignallingFileSystem() + try await fs.base.writeFile(path: "/a", data: Data("old".utf8), append: false) + try await fs.base.writeFile(path: "/unrelated", data: Data("before".utf8), append: false) + let workspace = Workspace(fileSystem: fs) + let checkpoint = try await workspace.createCheckpoint() + let events = await workspace.events(.init(includeCheckpoints: false)) + try await fs.base.writeFile(path: "/a", data: Data("new".utf8), append: false) + _ = try await workspace.restore(to: checkpoint) + guard case .changes? = await nextEditEvent(in: events) else { + Issue.record("missing restore event"); return + } + try await fs.base.writeFile(path: "/unrelated", data: Data("after".utf8), append: false) + fs.signal(.paths(["/unrelated": .entry])) + guard case let .changes(changes)? = await nextEditEvent(in: events) else { + Issue.record("missing unrelated external event"); return + } + #expect(changes.touchedPaths == ["/unrelated"]) + #expect(changes.textDiffOmissions.isEmpty) + #expect(changes.changes.first?.diff == TextDiff.lineBased(from: "before", to: "after")) + } + + @Test(arguments: [false, true]) + func `sparse restore failure recovers prior tree and preserves rollback error`(rollbackFails: Bool) async throws { + let base = InMemoryFileSystem() + for path: WorkspacePath in ["/a", "/b", "/unrelated"] { + try await base.writeFile(path: path, data: Data("old".utf8), append: false) + } + let fs = HookableFileSystem(base: base, forwardsPermissions: true) + let workspace = Workspace(fileSystem: fs) + let checkpoint = try await workspace.createCheckpoint() + for path: WorkspacePath in ["/a", "/b"] { + try await base.writeFile(path: path, data: Data("new".utf8), append: false) + } + let before = try await Snapshot.capture(from: base) + await fs.onWrite { _ in + await fs.onWrite { _ in + if rollbackFails { await fs.onWrite { _ in throw TestFileSystemError.injected } } + throw TestFileSystemError.injected + } + } + do { + _ = try await workspace.restore(to: checkpoint) + Issue.record("expected restore failure") + } catch { + #expect((error is WorkspaceRollbackError) == rollbackFails) + } + if !rollbackFails { #expect(try await Snapshot.capture(from: base).entry == before.entry) } + #expect(try await workspace.history().isEmpty) + #expect(try await workspace.checkpoints().count == 1) + #expect(await fs.writes.allSatisfy { $0 != "/unrelated" }) + } +} + +private extension Snapshot.Entry { + var fileBytes: Data? { + if case let .file(file) = self { return file.data } + return nil + } +} From 6238deb3633c45b786768131ac518ce955408e22 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 06:08:35 -0700 Subject: [PATCH 19/34] Serialize FSEvents callbacks and teardown on a private queue LocalFileSystem's FSEvents observation delivered callbacks on a concurrent global queue while cancellation stopped, invalidated and released the stream and its context box on whatever thread ended the subscription. A callback still in flight could then read the freed box (EXC_BAD_ACCESS in the event mapper), which crashed roughly one in four runs of a probe that tore down an observation while its tree was changing. The stream is now scheduled on a private serial queue and teardown is queued on that same queue, so it can never overlap a callback. The Linux inotify path, whose teardown was already serialized, now handles watch exhaustion and recovers watches once a directory becomes readable. A stress test creates and tears down observations in a loop while generating events. --- Sources/Workspace/FS/LocalFileSystem.swift | 24 ++++++--- .../Workspace/Support/FileChangeStreams.swift | 11 +++- .../ObservationLifetimeTests.swift | 51 +++++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) create mode 100644 Tests/WorkspaceTests/ObservationLifetimeTests.swift diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 0df18da..78cf7e0 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -1185,6 +1185,7 @@ enum FSEventsChangeStream { private final class Observation: @unchecked Sendable { private let stream: FSEventStreamRef private let box: Unmanaged + private let queue = DispatchQueue(label: "Workspace.FSEvents", qos: .utility) init(root: URL, resolvedRootPath: String, continuation: AsyncStream.Continuation) throws { box = .passRetained(Box(continuation, resolvedRootPath: resolvedRootPath)) @@ -1220,20 +1221,27 @@ enum FSEventsChangeStream { throw WorkspaceError.unsupported("could not create an FSEvents stream") } self.stream = stream - FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) + FSEventStreamSetDispatchQueue(stream, queue) guard FSEventStreamStart(stream) else { - FSEventStreamInvalidate(stream) - FSEventStreamRelease(stream) - box.release() + queue.sync { + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + box.release() + } throw WorkspaceError.unsupported("could not start an FSEvents stream") } } func cancel() { - FSEventStreamStop(stream) - FSEventStreamInvalidate(stream) - FSEventStreamRelease(stream) - box.release() + // The context is unretained during callbacks. Release it only after all + // callbacks on this serial queue finish, including when cancellation is + // triggered by the consumer while a callback is yielding an event. + queue.async { [self] in + FSEventStreamStop(stream) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + box.release() + } } } diff --git a/Sources/Workspace/Support/FileChangeStreams.swift b/Sources/Workspace/Support/FileChangeStreams.swift index ee959e5..9b3eeea 100644 --- a/Sources/Workspace/Support/FileChangeStreams.swift +++ b/Sources/Workspace/Support/FileChangeStreams.swift @@ -168,7 +168,8 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { } else { unknown = true } } } else { reconcile = true; unknown = true } - if isDirectory && structural { + // A directory can become watchable after a permission change. + if isDirectory && (structural || event.mask & UInt32(IN_ATTRIB) != 0) { reconcile = true } if event.mask & UInt32(IN_IGNORED | IN_DELETE_SELF | IN_MOVE_SELF) != 0 { @@ -234,6 +235,11 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { let watch = path.withCString { inotify_add_watch(descriptor, $0, mask) } if watch < 0 { let code = errno + // Resource exhaustion means coverage is incomplete, even if the root + // watch succeeded. End observation so accounting falls back to walks. + if code == ENOSPC || code == EMFILE || code == ENFILE || code == ENOMEM { + throw POSIXError(.init(rawValue: code) ?? .EIO) + } if path.utf8.elementsEqual(directory.path.utf8), requireRoot || (code != ENOENT && code != ENOTDIR) { throw POSIXError(.init(rawValue: code) ?? .EIO) } @@ -256,6 +262,9 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { // The parent may grant traversal without read access (for example 0711). // Its watch improves root-recreation coverage but cannot disable a valid root. if watch >= 0 { parentWatch = watch } + else if errno == ENOSPC || errno == EMFILE || errno == ENFILE || errno == ENOMEM { + throw POSIXError(.init(rawValue: errno) ?? .EIO) + } } } } diff --git a/Tests/WorkspaceTests/ObservationLifetimeTests.swift b/Tests/WorkspaceTests/ObservationLifetimeTests.swift new file mode 100644 index 0000000..0b4bd54 --- /dev/null +++ b/Tests/WorkspaceTests/ObservationLifetimeTests.swift @@ -0,0 +1,51 @@ +#if os(macOS) || os(Linux) +import Foundation +import Testing +@testable import Workspace + +@Suite("Native observation lifetime") +struct ObservationLifetimeTests { + @Test + func cancellationWhileCallbacksAreActive() async throws { + let root = try TestSupport.temporaryDirectory("ObservationLifetime") + defer { TestSupport.remove(root) } + let fs = try LocalFileSystem(root: root) + let events = try await withThrowingTaskGroup(of: Int.self) { group in + for worker in 0..<4 { + group.addTask { + var received = 0 + for _ in 0..<32 { + let stream = try await fs.changeSignals() + let consumer = Task { + var count = 0 + for await _ in stream { count += 1 } + return count + } + let producer = Task { + for i in 0..<32 { + try await fs.writeFile(path: WorkspacePath("/file-\(worker)-\(i)"), + data: Data(count: 32), append: false) + try await Task.sleep(for: .milliseconds(2)) + } + } + defer { consumer.cancel(); producer.cancel() } + // FSEvents has 40ms latency. Cancel with callbacks and more writes + // still in flight, then await termination before reusing the root. + try await Task.sleep(for: .milliseconds(55)) + consumer.cancel() + received += await consumer.value + try await producer.value + } + return received + } + } + var total = 0 + for try await count in group { total += count } + return total + } + // Exercise real callback delivery as well as 128 cancellation lifetimes. + #expect(events > 0) + #expect(try await fs.listDirectory(path: .root).count == 128) + } +} +#endif From 0c9f09294107fb58a9915f21053ca4672b6e313a Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 06:08:43 -0700 Subject: [PATCH 20/34] Cache LimitedFileSystem accounting and invalidate it from change signals LimitedFileSystem keeps its accounting (totals plus per-path kind, size, identity and link count, and alias indexes) between mutations instead of walking the whole tree on every preflight. Successful mutations reconcile the affected paths under the FIFO gate; failures and uncertain alias effects invalidate. When the base declares complete change coverage (changeSignalsAreComplete: LocalFileSystem and the forwarding wrappers; MountedFileSystem only when every layer does; never AuthorizedFileSystem, which filters signals), signalled scopes are invalidated and re-walked lazily. InMemoryFileSystem exposes an exact mutation generation instead. Other backends, and trees past an entry cap, keep walking as before. Accounting is as of the change signals processed before a projection finishes; revalidate() forces a fresh walk. On the medium benchmark fixture a write through LimitedFileSystem drops from about 21 ms to 0.19 ms, and on the large fixture from about 277 ms to 0.36 ms. --- .../CountingFileSystem.swift | 26 +- .../WorkspaceBenchmarks/Scenarios.swift | 5 + README.md | 46 +- Sources/Workspace/FS/FileSystem.swift | 18 + Sources/Workspace/FS/InMemoryFileSystem.swift | 53 +- Sources/Workspace/FS/LimitedFileSystem.swift | 483 +++++++++++++-- Sources/Workspace/FS/MountedFileSystem.swift | 33 +- Sources/Workspace/FS/SandboxFileSystem.swift | 3 +- .../FS/SecurityScopedFileSystem.swift | 3 +- Sources/Workspace/Support/Permissions.swift | 5 +- Tests/WorkspaceTests/LimitedCacheTests.swift | 568 ++++++++++++++++++ .../WorkspaceTests/LimitedCoverageTests.swift | 27 + 12 files changed, 1189 insertions(+), 81 deletions(-) create mode 100644 Tests/WorkspaceTests/LimitedCacheTests.swift create mode 100644 Tests/WorkspaceTests/LimitedCoverageTests.swift diff --git a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift index 3f9beac..734c180 100644 --- a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift +++ b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift @@ -13,8 +13,24 @@ struct IOCounts: Codable, Equatable, Sendable { } /// A lock only protects counter updates; no lock is held across an await. -class CountingFileSystem: FileSystem, TraversingFileSystem, ExactTimestampFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { +class CountingFileSystem: CompleteFileSystemChangeSource, MutationGenerationFileSystem, TraversingFileSystem, ExactTimestampFileSystem, PreviewSemanticsFileSystem, PotentialPathResolvingFileSystem, @unchecked Sendable { + var changeSignalsAreComplete: Bool { get async { await base.changeSignalsAreComplete } } var hasExactContentTimestamps: Bool { get async { await base.supportsExactContentTimestamps } } + var mutationGeneration: UInt64? { + get async { await (base as? any MutationGenerationFileSystem)?.mutationGeneration } + } + func changeSignals() async throws -> AsyncStream { + guard let source = base as? any FileSystemChangeSource else { + throw BenchmarkError.message("backend does not provide filesystem notifications") + } + return try await source.changeSignals() + } + func changes() async throws -> AsyncStream { + guard let source = base as? any FileSystemChangeSource else { + throw BenchmarkError.message("backend does not provide filesystem notifications") + } + return try await source.changes() + } let base: any FileSystem private let lock = NSLock() private var counts = IOCounts() @@ -167,18 +183,18 @@ class CountingFileSystem: FileSystem, TraversingFileSystem, ExactTimestampFileSy } } -/// Only Local advertises observation; all filesystem forwarding is inherited. -final class ObservedCountingFileSystem: CountingFileSystem, FileSystemChangeSource, +/// Adds observation-call counters for watcher scenarios; forwarding remains unchanged. +final class ObservedCountingFileSystem: CountingFileSystem, @unchecked Sendable { - func changeSignals() async throws -> AsyncStream { + override func changeSignals() async throws -> AsyncStream { call("changeSignals") guard let source = base as? any FileSystemChangeSource else { throw BenchmarkError.message("backend does not provide filesystem notifications") } return try await source.changeSignals() } - func changes() async throws -> AsyncStream { + override func changes() async throws -> AsyncStream { call("changes") guard let source = base as? any FileSystemChangeSource else { throw BenchmarkError.message("backend does not provide filesystem notifications") diff --git a/Benchmarks/WorkspaceBenchmarks/Scenarios.swift b/Benchmarks/WorkspaceBenchmarks/Scenarios.swift index 59d931e..e501c5e 100644 --- a/Benchmarks/WorkspaceBenchmarks/Scenarios.swift +++ b/Benchmarks/WorkspaceBenchmarks/Scenarios.swift @@ -113,6 +113,11 @@ struct ScenarioRunner { maxTotalBytes: UInt64(fixture.bytes) * 2, maxEntryCount: fixture.fileCount * 3 ) ) : counter + // limitedWrite measures a warm accounting cache. Workspace's no-op priming + // does not reach the filesystem mutation boundary, so prime it explicitly. + if operation == .limitedWrite { + try await filesystem.writeFile(path: path, data: original, append: false) + } let store = scratch.appendingPathComponent(storeName) defer { try? FileManager.default.removeItem(at: store) } let persistence: Workspace.Persistence = diff --git a/README.md b/README.md index 1f975b6..e954bc7 100644 --- a/README.md +++ b/README.md @@ -491,6 +491,50 @@ let authorized = AuthorizedFileSystem(base: limited, authorizer: rules) let workspace = Workspace(fileSystem: authorized) ``` +`LimitedFileSystem` serializes its own preflight and mutation with a FIFO gate. Quotas +count logical regular-file bytes per visible path (including hard-link aliases), and +all non-root entries, including dangling symlinks. Appends charge aliases by identity +and link count; unknown aliases are charged conservatively. Already over-quota trees +may shrink or preserve usage, but cannot increase an exceeded quota. Accounting reads +metadata only and resolves symlinks in the backend's namespace, including mounts. + +When the base declares complete change-signal coverage, Limited caches totals, compact +per-path accounting metadata (kind, size, identity and link count), children, +and alias indexes. Successful mutations refresh their affected metadata under the gate; +a failed or uncertain mutation discards the cache. File and directory identities are +needed to rule out unrecognized aliases through compositions (even for single-link +files); missing identities make the mutation uncertain and force a fresh walk on the +next preflight. External `.entry`, `.shallow`, and +`.recursive` signals invalidate those scopes for lazy reconciliation on the next +preflight. Removed/replaced directories invalidate their former descendants; hard-link +aliases are refreshed together. `.rescan` discards all cached accounting. A failed or +ended subscription falls back to a full metadata walk on every preflight, unless an +exact mutation generation is available. Coverage is checked at every preflight; Mounted +declares complete coverage only when its base and every mount do, including later mounts. +Incomplete or undeclared coverage uses full walks (or the exact generation capability). +Each Limited over Local registers its own recursive inotify watcher on Linux; large trees +can exhaust the watch limit (ENOSPC), at which point accounting safely falls back to walks. +The retained cache is capped at 250,000 entries (25,000 on iOS/watchOS); above that cap, +accounting is discarded after the operation and every preflight walks the tree. + +**Consistency:** a quota decision incorporates signals processed by this Limited actor +before its projection finishes. Signals buffered in the backend or stream, or queued +for actor processing, may arrive after a decision, so external writes can temporarily make accounting stale and +can exceed quotas. `try await limited.revalidate()` forces a fresh metadata walk under +the gate without waiting for signals. With external writers quiescent, it closes the +notification-delivery gap. Neither revalidation nor the gate makes preflight atomic +with writers using another wrapper or the base directly. The package-level `changeSignalsAreComplete` +capability is an explicit guarantee of coverage for all visible children; sources that +only provide a partial stream cannot enable notification-based accounting reuse. + +Without complete change signals, each preflight walks the whole tree. The package's optional +exact mutation-generation capability lets InMemoryFileSystem reuse accounting when +unchanged; direct mutations, reset, and partial failures change its generation and +force a new walk. Generation changes during a scan or this wrapper's mutation prevent +reuse of that potentially inconsistent snapshot. +Accounting retains strict bulk-walk semantics: a listed entry vanishing before metadata +lookup fails the preflight and invalidates partial accounting; the next preflight scans anew. + Authorization supports one-shot, duration-limited, and session approvals plus bounded audit records. Prefix rules use path-component boundaries and require both paths to match for copy and move. Local scans record symlinks without descending into them and retain per-entry authorization @@ -594,7 +638,7 @@ deadlines (30 minutes per child by default). | Scenario | What's timed | Setup excluded | Counter blind spots | | --- | --- | --- | --- | -| `writeText`, `apply50`, `limitedWrite` | One recorded write, 50 edits, quota-limited write | No-op preparation, file reset | Checkpoint store; Limited metadata traversal is visible | +| `writeText`, `apply50`, `limitedWrite` | One recorded write, 50 edits, quota-limited write with warm accounting | No-op preparation, file reset, Limited accounting priming | Checkpoint store; Limited metadata traversal is visible | | `transaction` / `transactionFirst` | Begin, one draft write, commit | Preparation; initial begin/discard and baseline checkpoint for steady-state `transaction` only | Draft filesystem, checkpoint store | | `preview` | One edit preview | Preparation | Draft filesystem | | `checkpointFirst` / `checkpointIncremental` | First capture / capture after one change | Preparation; baseline and change for incremental | Checkpoint store | diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index 9c7633c..7ea3edc 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -217,6 +217,24 @@ public protocol FileSystemChangeSource: FileSystem { func changeSignals() async throws -> AsyncStream } +/// Opt-in guarantee that the stream covers every visible child. A composite must +/// re-evaluate this after topology changes and terminate a stream if coverage fails. +package protocol CompleteFileSystemChangeSource: FileSystemChangeSource { + var changeSignalsAreComplete: Bool { get async } +} + +extension FileSystem { + package var changeSignalsAreComplete: Bool { + get async { await (self as? any CompleteFileSystemChangeSource)?.changeSignalsAreComplete ?? false } + } +} + +#if os(macOS) || os(Linux) +extension LocalFileSystem: CompleteFileSystemChangeSource { + package var changeSignalsAreComplete: Bool { true } +} +#endif + /// Invalidates paths in the change source's own filesystem namespace. /// Include deleted or replaced ancestors so their former descendants are checked. /// Use `.rescan` when paths are unknown, events were dropped, or coverage is uncertain. diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index 6f029ad..99c85f3 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -13,7 +13,10 @@ import Glibc /// /// Actor isolation serializes access to the tree so the filesystem can be shared safely across /// concurrent tasks. -public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, ExactTimestampFileSystem, TraversingFileSystem { +public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, ExactTimestampFileSystem, TraversingFileSystem, MutationGenerationFileSystem { + private var generation: UInt64 = 0 + package var mutationGeneration: UInt64? { generation } + package nonisolated var hasExactContentTimestamps: Bool { true } private var lastTimestamp = Date().timeIntervalSinceReferenceDate @@ -78,12 +81,14 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/configure(rootDirectory:)``. func configure(rootDirectory: URL) async throws { + defer { generation &+= 1 } _ = rootDirectory root = makeNode(kind: .directory([:]), permissions: 0o755) } /// Clears the filesystem and recreates an empty root directory. public func reset() { + defer { generation &+= 1 } root = makeNode(kind: .directory([:]), permissions: 0o755) } @@ -91,7 +96,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E // caller's executor; only immutable metadata/content values cross that boundary. private final class WalkCursor { enum Frame { - case entry(WorkspacePath, Node?, TimeInterval?) + case entry(WorkspacePath, Node?, UInt64?) case directory(FileInfo) } var frames: [Frame] @@ -99,7 +104,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E } private struct WalkEntry: Sendable { var info: FileInfo - var capturedAt: TimeInterval + var capturedAt: UInt64 var data: Data? var target: String? } @@ -140,13 +145,13 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E try Task.checkCancellation() switch frame { case let .directory(info): - return WalkEntry(info: info, capturedAt: lastTimestamp) + return WalkEntry(info: info, capturedAt: generation) case let .entry(path, listed, listedAt): let entry: Node do { - // Every mutation advances lastTimestamp before yielding. After + // Every mutation attempt advances generation before yielding. After // reentrancy, resolve the listed path again, never detached nodes. - if let listed, listedAt == lastTimestamp { entry = listed } + if let listed, listedAt == generation { entry = listed } else { entry = try node(at: path, followFinalSymlink: false) } } catch { guard (listed == nil || skippingVanishedEntries), Snapshot.isMissingEntry(error) else { throw error } @@ -160,28 +165,28 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E (name, child, walkSortKey(name: name.string, isDirectory: child.isDirectory)) }.sorted { $0.2.utf8.lexicographicallyPrecedes($1.2.utf8) } for (name, child, _) in ordered.reversed() { - cursor.frames.append(.entry(path.appending(name.string), child, lastTimestamp)) + cursor.frames.append(.entry(path.appending(name.string), child, generation)) } case let .file(data): - return WalkEntry(info: info, capturedAt: lastTimestamp, data: data) + return WalkEntry(info: info, capturedAt: generation, data: data) case let .symlink(target): - return WalkEntry(info: info, capturedAt: lastTimestamp, target: target) + return WalkEntry(info: info, capturedAt: generation, target: target) } } } return nil } - private func readWalkFile(at path: WorkspacePath, capturedAt: TimeInterval, data: Data?, length: Int?) async throws -> Data { + private func readWalkFile(at path: WorkspacePath, capturedAt: UInt64, data: Data?, length: Int?) async throws -> Data { if let length, length < 0 { throw WorkspaceError.unsupported("read length must not be negative") } - guard capturedAt == lastTimestamp, let data else { + guard capturedAt == generation, let data else { return try await readFile(path: path, offset: 0, length: length) } return length.map { Data(data.prefix($0)) } ?? data } - private func readWalkLink(at path: WorkspacePath, capturedAt: TimeInterval, target: String?) async throws -> String { - guard capturedAt == lastTimestamp, let target else { return try await readSymlink(path: path) } + private func readWalkLink(at path: WorkspacePath, capturedAt: UInt64, target: String?) async throws -> String { + guard capturedAt == generation, let target else { return try await readSymlink(path: path) } return target } @@ -227,14 +232,20 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/createFile(path:data:)``. Actor isolation makes the existence check and /// write a single atomic step. public func createFile(path: WorkspacePath, data: Data) async throws { + defer { generation &+= 1 } if (try? node(at: path, followFinalSymlink: false)) != nil { throw posixError(EEXIST) } - try await writeFile(path: path, data: data, append: false) + try write(path: path, data: data, append: false) } /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + defer { generation &+= 1 } + try write(path: path, data: data, append: append) + } + + private func write(path: WorkspacePath, data: Data, append: Bool) throws { guard !path.isRoot else { throw posixError(EISDIR) } @@ -266,6 +277,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/createDirectory(path:recursive:)``. public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { + defer { generation &+= 1 } if path.isRoot { return } @@ -303,6 +315,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/remove(path:recursive:)``. public func remove(path: WorkspacePath, recursive: Bool) async throws { + defer { generation &+= 1 } if path.isRoot { throw posixError(EPERM) } @@ -324,6 +337,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/move(from:to:)``. public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { + defer { generation &+= 1 } if sourcePath == destinationPath { return } @@ -370,6 +384,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + defer { generation &+= 1 } let sourceNode = try node(at: sourcePath, followFinalSymlink: false) if sourceNode.isDirectory, !recursive { throw posixError(EISDIR) @@ -388,6 +403,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/createSymlink(path:target:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { + defer { generation &+= 1 } _ = try WorkspacePath(validating: target, relativeTo: path.dirname) guard !path.isRoot else { throw posixError(EEXIST) @@ -406,6 +422,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { + defer { generation &+= 1 } guard !path.isRoot else { throw posixError(EEXIST) } @@ -440,6 +457,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/setPermissions(path:permissions:)``. public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { + defer { generation &+= 1 } let node = try node(at: path, followFinalSymlink: false) node.permissions = Int(permissions.rawValue) node.modificationDate = nextTimestamp() @@ -674,3 +692,10 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E extension InMemoryFileSystem: PreviewSemanticsFileSystem { package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { .init() } } + +/// Optional exact revision for metadata accounting. Each attempted public mutation advances +/// once, including failures; nil means unavailable. Forwarders must preserve that rule. +/// Readers compare before/after a scan and before/after their own mutation. +package protocol MutationGenerationFileSystem: FileSystem { + var mutationGeneration: UInt64? { get async } +} diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index f1a7e70..fb9d3d3 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -34,11 +34,30 @@ public enum FileSystemLimitError: Error, Sendable, Equatable, CustomStringConver } } +// Retained accounting omits timestamps, permissions and the duplicate path. +struct LimitedAccountingEntry: Sendable { + let kind: FileTree.Kind + let size: UInt64 + let identity: FileIdentity? + let linkCount: UInt64? + + init(_ info: FileInfo) { + kind = info.kind + size = info.size + identity = info.identity + linkCount = info.linkCount + } +} + /// A serializing filesystem decorator that rejects mutations exceeding projected limits. /// /// All mutations through this instance share a FIFO gate. Mutations made directly to the base, -/// or through another decorator, are outside that gate. Accounting scans metadata, never content, -/// on each mutation so sequential external changes are reflected without a stale usage cache. +/// or through another decorator, are outside that gate. Accounting uses metadata, never content. +/// With complete change notifications, preflights reconcile signals consumed before projection finishes; +/// undelivered external changes may remain invisible. `revalidate()` forces a fresh metadata +/// walk without waiting for notifications. Neither notifications nor revalidation make a +/// preflight atomic with external writers. Without notifications or an exact mutation generation, +/// every preflight walks the whole tree. Successful mutations reconcile only affected metadata. /// Append accounting uses metadata identities and link counts. Unknown aliases are charged /// conservatively as all regular files of the target's size. Without the replacement feature, /// non-appending writes are projected as replacing the final link itself; this is conservative @@ -50,6 +69,20 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti private let base: any FileSystem private let pathResolver: FileSystemPathResolver public let limits: FileSystemLimits + // Above this bound the same projection runs on an ephemeral full walk. + #if os(iOS) || os(watchOS) + static let accountingCacheEntryLimit = 25_000 + #else + static let accountingCacheEntryLimit = 250_000 + #endif + private var accounting = Accounting() + private var cacheValid = false + private var cachedGeneration: UInt64? + private var observationStarted = false + private var observationAvailable = false + private var observationTask: Task? + private var invalidations: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] + private var rescan = false private var mutationActive = false private var mutationWaiters: [CheckedContinuation] = [] @@ -59,6 +92,17 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti self.limits = limits } + deinit { observationTask?.cancel() } + + /// Forces fresh metadata accounting under the mutation gate. External writers must be + /// quiescent for a coherent result; this does not wait for backend notification delivery. + public func revalidate() async throws { + await acquireMutation() + defer { discardOversizedAccounting(); releaseMutation() } + cacheValid = false + _ = try await prepareAccounting() + } + nonisolated package var supportsScopedWalk: Bool { base.supportsScopedWalk } nonisolated(nonsending) package func walk(at path: WorkspacePath, skippingUnreadableDirectories: Bool, skippingVanishedEntries: Bool = false, @@ -147,9 +191,34 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { await acquireMutation() - defer { releaseMutation() } + defer { discardOversizedAccounting(); releaseMutation() } try Task.checkCancellation() - try await base.setPermissions(path: path, permissions: permissions) + let generation = cachedGeneration + do { + let behavior = await base.previewBehavior(at: path) + let target = try await pathResolver.resolve(path, features: await base.capabilities(), + followFinal: behavior.followsPermissionSymlinks) + try await base.setPermissions(path: path, permissions: permissions) + guard cacheValid else { return } + do { + // Permissions may change traversal of this subtree, including its aliases. + try await reconcile([target: .recursive]) + let after = await (base as? any MutationGenerationFileSystem)?.mutationGeneration + if let generation { cacheValid = after == generation &+ 1 } + cachedGeneration = after + if accounting.unidentifiedEntries > 0 { cacheValid = false } + } catch { cacheValid = false } + } catch { + cacheValid = false + throw error + } + } + + private func discardOversizedAccounting() { + if accounting.tree.count > Self.accountingCacheEntryLimit { + accounting = Accounting() + cacheValid = false + } } private func acquireMutation() async { @@ -181,12 +250,40 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti private func mutate(_ mutation: Mutation, operation: () async throws -> Void) async throws { await acquireMutation() - defer { releaseMutation() } + defer { discardOversizedAccounting(); releaseMutation() } try Task.checkCancellation() - if limits.maxTotalBytes != nil || limits.maxEntryCount != nil { - try await preflight(mutation) + guard limits.maxTotalBytes != nil || limits.maxEntryCount != nil else { + try await operation() + return + } + let retain = try await prepareAccounting() + let affected = try await preflight(mutation) + // Missing identities can hide aliases exposed through a composition, even + // when a file has a single physical link. Treat that effect as uncertain. + let uncertain = accounting.unidentifiedEntries > 0 + let generation = cachedGeneration + do { + try await operation() + } catch { + cacheValid = false // Includes partial mutation and cancellation after backend work. + throw error + } + guard retain, !uncertain else { cacheValid = false; return } + do { + // Read back the projected scopes so identities, link counts, implicit parents and + // backend-specific copy/replacement behavior are exact, rather than invented. + try await reconcile(affected) + let after = await (base as? any MutationGenerationFileSystem)?.mutationGeneration + if let generation { + cacheValid = after == generation &+ 1 + } + cachedGeneration = after + if accounting.unidentifiedEntries > 0 { cacheValid = false } + } catch { + // The mutation succeeded. A failed metadata refresh must not turn it into a + // reported mutation failure (and tempt the caller to retry it). + cacheValid = false } - try await operation() } private func checkWrite(_ count: Int) throws { @@ -196,51 +293,324 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti } private struct Usage { - var bytes: UInt64 = 0 + // Saturate only at the public quota boundary; subtraction must remain exact even + // when a backend reports logical sizes whose sum exceeds UInt64.max. + private var lowBytes: UInt64 = 0 + private var highBytes: UInt64 = 0 + var bytes: UInt64 { highBytes == 0 ? lowBytes : UInt64.max } var entries = 0 - mutating func add(bytes amount: UInt64) { - let sum = bytes.addingReportingOverflow(amount) - bytes = sum.overflow ? UInt64.max : sum.partialValue + let sum = lowBytes.addingReportingOverflow(amount) + lowBytes = sum.partialValue + if sum.overflow { highBytes += 1 } + } + mutating func add(_ other: Usage) { + add(bytes: other.lowBytes) + highBytes += other.highBytes + entries += other.entries + } + mutating func subtract(_ other: Usage) { + let difference = lowBytes.subtractingReportingOverflow(other.lowBytes) + lowBytes = difference.partialValue + highBytes -= other.highBytes + (difference.overflow ? 1 : 0) + entries -= other.entries + } + } + + private struct Accounting { + var tree: [WorkspacePath: LimitedAccountingEntry] = [:] + var children: [WorkspacePath: Set] = [:] + var identities: [FileIdentity: Set] = [:] + var filesBySize: [UInt64: Set] = [:] + var unknownFilesBySize: [UInt64: Set] = [:] + var usage = Usage() + var unidentifiedEntries = 0 + + mutating func put(_ info: FileInfo) { + erase(info.path) + tree[info.path] = LimitedAccountingEntry(info) + if info.kind != .symlink, info.identity == nil { unidentifiedEntries += 1 } + if !info.path.isRoot { + children[info.path.dirname, default: []].insert(info.path) + usage.entries += 1 + } + if let identity = info.identity { identities[identity, default: []].insert(info.path) } + if info.kind == .file { + filesBySize[info.size, default: []].insert(info.path) + if info.identity == nil, info.linkCount != 1 { + unknownFilesBySize[info.size, default: []].insert(info.path) + } + usage.add(bytes: info.size) + } + } + + mutating func erase(_ path: WorkspacePath) { + guard let old = tree.removeValue(forKey: path) else { return } + if old.kind != .symlink, old.identity == nil { unidentifiedEntries -= 1 } + if !path.isRoot { + children[path.dirname]?.remove(path) + if children[path.dirname]?.isEmpty == true { children.removeValue(forKey: path.dirname) } + usage.entries -= 1 + } + if let identity = old.identity { + identities[identity]?.remove(path) + if identities[identity]?.isEmpty == true { identities.removeValue(forKey: identity) } + } + if old.kind == .file { + filesBySize[old.size]?.remove(path) + if filesBySize[old.size]?.isEmpty == true { filesBySize.removeValue(forKey: old.size) } + unknownFilesBySize[old.size]?.remove(path) + if unknownFilesBySize[old.size]?.isEmpty == true { unknownFilesBySize.removeValue(forKey: old.size) } + var removed = Usage() + removed.add(bytes: old.size) + usage.subtract(removed) + } + } + + func subtree(_ path: WorkspacePath) -> Set { + var result: Set = [] + var pending = [path] + while let next = pending.popLast() { + if tree[next] != nil { result.insert(next) } + pending.append(contentsOf: children[next] ?? []) + } + return result + } + + func aliases(of path: WorkspacePath) -> Set { + guard let info = tree[path] else { return [] } + if let identity = info.identity { + var result = identities[identity] ?? [path] + if info.kind == .file, info.linkCount != 1 { + result.formUnion(unknownFilesBySize[info.size] ?? []) + } + return result + } + guard info.kind == .file, info.linkCount != 1 else { return [path] } + return Set((filesBySize[info.size] ?? []).filter { tree[$0]?.linkCount != 1 }) + } + } + + private func startObservation() async { + guard !observationStarted else { return } + observationStarted = true + guard let source = base as? any FileSystemChangeSource else { return } + do { + let stream = try await source.changeSignals() + observationAvailable = true + observationTask = Task { [weak self] in + for await signal in stream { + guard !Task.isCancelled else { return } + await self?.receive(signal) + } + await self?.observationEnded() + } + } catch { + observationEnded() + } + } + + private func receive(_ signal: FileSystemChangeSignal) { + switch signal { + case .rescan: + rescan = true + invalidations.removeAll() + case let .paths(paths): + guard !rescan else { return } + for (path, scope) in paths { invalidations[path] = max(invalidations[path] ?? .entry, scope) } + } + } + + private func observationEnded() { + observationAvailable = false + rescan = true + invalidations.removeAll() + } + + private func prepareAccounting() async throws -> Bool { + await startObservation() + while true { + try Task.checkCancellation() + let generation = await (base as? any MutationGenerationFileSystem)?.mutationGeneration + // Mounted coverage can change after a later mount. A live stream alone + // does not guarantee notifications from every child. + let complete = await base.changeSignalsAreComplete + let retain = (observationAvailable && complete) || generation != nil + if !retain || generation != cachedGeneration { cacheValid = false } + let full = !cacheValid || rescan + let dirty = invalidations + invalidations.removeAll() + rescan = false + do { + if full { + accounting = Accounting() + try await capture(.root) + cacheValid = true + } else { + try await reconcile(dirty) + } + } catch { + cacheValid = false + throw error + } + let after = await (base as? any MutationGenerationFileSystem)?.mutationGeneration + if generation != after { cacheValid = false; continue } + cachedGeneration = after + if rescan || !invalidations.isEmpty { continue } + return retain && accounting.tree.count <= Self.accountingCacheEntryLimit } } - private func preflight(_ mutation: Mutation) async throws { - var tree: [WorkspacePath: FileInfo] = [:] + private func capture(_ path: WorkspacePath, info: FileInfo? = nil) async throws { if base.supportsScopedWalk { - try await base.walk { info, _, _ in tree[info.path] = info } + // Preserve strict accounting-walk failures for vanished listed entries. + // Callers invalidate partial accounting before propagating the error. + try await base.walk(at: path, skippingVanishedEntries: false) { info, _, _ in self.accounting.put(info) } } else { - // Preserve metadata-only accounting for custom backends: listings already - // supply child metadata and need not permit separate child stat calls. - var pending = [try await base.stat(path: .root)] - while let info = pending.popLast() { - tree[info.path] = info - if info.kind == .directory { - for child in try await base.listDirectory(path: info.path) { + // Listings supply metadata: no child stat or content-read requirement. + let root: FileInfo? + if let info { root = info } + else if path.isRoot { root = try await base.stat(path: path) } + else { root = try await Snapshot.infoIfPresent(at: path, on: base) } + guard let root else { return } + var pending = [root] + while let next = pending.popLast() { + accounting.put(next) + if next.kind == .directory { + for child in try await base.listDirectory(path: next.path) { var metadata = child.info - metadata.path = info.path.appending(child.name) + metadata.path = next.path.appending(child.name) pending.append(metadata) } } } } - let current = Self.usage(tree) + } + + private func reconcile(_ scopes: [WorkspacePath: FileSystemChangeSignal.Scope]) async throws { + // Expand old aliases before removing metadata. Directory aliases can expose a + // mutation at another mount point; file aliases can change size/link count too. + var expanded = scopes + for (path, scope) in scopes { + for alias in accounting.aliases(of: path) { + expanded[alias] = max(expanded[alias] ?? .entry, scope) + } + } + // Parents precede their descendants, including entry-scoped replacements. + for (path, scope) in expanded.sorted(by: { $0.key < $1.key }) { + var ancestor = path + var covered = false + while !ancestor.isRoot { + ancestor = ancestor.dirname + if expanded[ancestor] == .recursive { covered = true; break } + } + if !covered { try await refresh(path, scope: scope) } + } + } + + private func refresh(_ path: WorkspacePath, scope: FileSystemChangeSignal.Scope, + includingAliases: Bool = true, listedInfo: FileInfo? = nil) async throws { + // A delayed child event must not resurrect entries beneath a directory that + // has since become a symlink. Accounting walks never descend through links. + if !path.isRoot { + var parent = path.dirname + while true { + guard accounting.tree[parent]?.kind == .directory else { return } + if parent.isRoot { break } + parent = parent.dirname + } + } + let old = accounting.tree[path] + let info: FileInfo? + if let listedInfo { info = listedInfo } + else { info = try await Snapshot.infoIfPresent(at: path, on: base) } + let sameDirectory = old?.kind == .directory && info?.kind == .directory + && old?.identity != nil && old?.identity == info?.identity + if scope == .recursive || (old?.kind == .directory && !sameDirectory) + || (info?.kind == .directory && !sameDirectory) { + // Also refresh aliases outside a removed/replaced subtree: unlink changes + // their link counts and a later append must not use stale singleton data. + let removed = accounting.subtree(path) + var aliases: Set = [] + if includingAliases { + for entry in removed { aliases.formUnion(accounting.aliases(of: entry)) } + } + for entry in removed { accounting.erase(entry) } + if let info { try await capture(path, info: info) } + for alias in aliases.subtracting(removed) { + try await refresh(alias, scope: .entry, includingAliases: false) + } + return + } + if let info { + accounting.put(info) + if includingAliases, info.kind != .directory { + for alias in accounting.aliases(of: path) where alias != path { + try await refresh(alias, scope: .entry, includingAliases: false) + } + } + } else { accounting.erase(path) } + if scope == .shallow, info?.kind == .directory { + var former = accounting.children[path] ?? [] + for child in try await base.listDirectory(path: path) { + let childPath = path.appending(child.name) + former.remove(childPath) + // A shallow scope validates immediate children but retains unchanged + // directory interiors. Identity-less directories cannot prove reuse. + var metadata = child.info + metadata.path = childPath + try await refresh(childPath, scope: .entry, listedInfo: metadata) + } + for child in former { try await refresh(child, scope: .recursive) } + } + } + + private func preflight(_ mutation: Mutation) async throws -> [WorkspacePath: FileSystemChangeSignal.Scope] { + while true { + do { + let affected = try await project(mutation) + if !rescan && invalidations.isEmpty { return affected } + } catch { + // Even an early projection rejection (e.g. EEXIST) may have raced a + // signal consumed during path resolution. Retry against fresh metadata. + if !rescan && invalidations.isEmpty { throw error } + } + _ = try await prepareAccounting() + } + } + + private func project(_ mutation: Mutation) async throws -> [WorkspacePath: FileSystemChangeSignal.Scope] { + let tree = accounting.tree + let current = accounting.usage var projected = current + var affected: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] let features = await base.capabilities() - func resolve(_ path: WorkspacePath, in tree: [WorkspacePath: FileInfo], followFinal: Bool) async throws -> WorkspacePath { + func resolve(_ path: WorkspacePath, in tree: [WorkspacePath: LimitedAccountingEntry], followFinal: Bool) async throws -> WorkspacePath { try await pathResolver.resolve(path, in: tree, features: features, followFinal: followFinal) } func subtract(_ path: WorkspacePath) { - let removed = Self.usage(tree.filter { Self.contains($0.key, in: path) }) - projected.bytes -= removed.bytes - projected.entries -= removed.entries + let removed = usage(of: accounting.subtree(path)) + projected.subtract(removed) + affected[path] = .recursive } func addParents(_ path: WorkspacePath) { var parent = path.dirname - while !parent.isRoot { - if tree[parent] == nil { projected.entries += 1 } + var changedRoot = path + while true { + if !parent.isRoot, tree[parent] == nil { + projected.entries += 1 + affected[parent] = .recursive + changedRoot = parent + } + // Translate only the changed suffix through directory aliases. Walking + // the entire aliased directory would make a leaf write scale with its tree. + for alias in accounting.aliases(of: parent) where alias != parent { + let suffix = changedRoot.components.dropFirst(parent.components.count).joined(separator: "/") + affected[alias.appending(suffix)] = .recursive + } + if parent.isRoot { break } parent = parent.dirname } } @@ -269,13 +639,15 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti let aliases: UInt64 if info.linkCount == 1, info.identity == nil { aliases = 1 + } else if let identity = info.identity { + let known = accounting.identities[identity]?.count ?? 1 + let uncertain = info.linkCount == 1 ? 0 : (accounting.unknownFilesBySize[info.size]?.count ?? 0) + aliases = UInt64(known + uncertain) } else { - aliases = UInt64(tree.reduce(0) { count, entry in - let candidate = entry.value - guard candidate.kind == .file, candidate.size == info.size else { return count } - if entry.key == target { return count + 1 } + aliases = UInt64((accounting.filesBySize[info.size] ?? []).reduce(0) { count, path in + guard let candidate = tree[path] else { return count } + if path == target { return count + 1 } if let identity = info.identity, let other = candidate.identity { - // A mount can expose even a single-link inode at multiple paths. return identity == other ? count + 1 : count } if info.linkCount == 1 || candidate.linkCount == 1 { return count } @@ -285,36 +657,39 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti let growth = size.multipliedReportingOverflow(by: aliases) projected.add(bytes: growth.overflow ? UInt64.max : growth.partialValue) } + affected[target] = .entry } else { addLeaf(target, bytes: size) } case let .directory(path): let target = try await resolve(path, in: tree, followFinal: true) if tree[target] == nil { addLeaf(target, bytes: 0) } + else { affected[target] = .entry } case let .remove(path): let target = try await resolve(path, in: tree, followFinal: false) subtract(target) + addParents(target) case let .move(source, destination), let .copy(source, destination): let from = try await resolve(source, in: tree, followFinal: false) let to = try await resolve(destination, in: tree, followFinal: false) guard tree[from] != nil else { throw Self.posix(ENOENT) } if from == to { break } - let copied = Self.usage(tree.filter { Self.contains($0.key, in: from) }) + let copied = usage(of: accounting.subtree(from)) + affected[to] = .recursive if case .move = mutation { if tree[from]?.kind == .directory, Self.contains(to, in: from) { throw Self.posix(EINVAL) } // Source and destination subtrees can overlap on an invalid backend move. // Subtract their union once; let the backend preserve its destination error. - let removed = Self.usage(tree.filter { - Self.contains($0.key, in: to) || Self.contains($0.key, in: from) - }) - projected.bytes -= removed.bytes - projected.entries -= removed.entries + let removed = usage(of: accounting.subtree(to).union(accounting.subtree(from))) + projected.subtract(removed) + affected[from] = .recursive + addParents(from) } else { subtract(to) } addParents(to) - projected.add(bytes: copied.bytes) - projected.entries += copied.entries + (from.isRoot && !to.isRoot ? 1 : 0) + projected.add(copied) + projected.entries += (from.isRoot && !to.isRoot ? 1 : 0) case let .symlink(path): let target = try await resolve(path, in: tree, followFinal: false) if tree[target] != nil { throw Self.posix(EEXIST) } @@ -326,6 +701,7 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti guard let info = tree[from] else { throw Self.posix(ENOENT) } if info.kind == .directory { throw Self.posix(EPERM) } addLeaf(target, bytes: info.kind == .file ? info.size : 0) + affected[from] = .entry } if let limit = limits.maxTotalBytes, projected.bytes > limit, projected.bytes > current.bytes { throw FileSystemLimitError.totalBytes(attempted: projected.bytes, limit: limit) @@ -333,11 +709,13 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti if let limit = limits.maxEntryCount, projected.entries > limit, projected.entries > current.entries { throw FileSystemLimitError.entryCount(attempted: projected.entries, limit: limit) } + return affected } - private static func usage(_ tree: [WorkspacePath: FileInfo]) -> Usage { + private func usage(of paths: Set) -> Usage { var result = Usage() - for (path, info) in tree { + for path in paths { + guard let info = accounting.tree[path] else { continue } if !path.isRoot { result.entries += 1 } if info.kind == .file { result.add(bytes: info.size) } } @@ -363,7 +741,7 @@ struct FileSystemPathResolver: Sendable { } func resolve( - _ path: WorkspacePath, in tree: [WorkspacePath: FileInfo]? = nil, + _ path: WorkspacePath, in tree: [WorkspacePath: LimitedAccountingEntry]? = nil, features: FileSystemFeatures, followFinal: Bool, depth: Int = 0 ) async throws -> WorkspacePath { guard depth < 64 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ELOOP)) } @@ -403,10 +781,10 @@ struct FileSystemPathResolver: Sendable { var current = WorkspacePath.root for (index, component) in path.components.enumerated() { current = current.appending(component) - let info: FileInfo? - if let tree { info = tree[current] } - else { info = try? await base.stat(path: current) } - if info?.kind == .symlink { + let kind: FileTree.Kind? + if let tree { kind = tree[current]?.kind } + else { kind = try? await base.stat(path: current).kind } + if kind == .symlink { let target = try await base.readSymlink(path: current) let targetPath = try WorkspacePath(validating: target, relativeTo: current.dirname) let suffix = path.components.dropFirst(index + 1).joined(separator: "/") @@ -415,7 +793,7 @@ struct FileSystemPathResolver: Sendable { features: features, followFinal: true, depth: depth + 1 ) } - if index < path.components.count - 1, let info, info.kind != .directory { + if index < path.components.count - 1, let kind, kind != .directory { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTDIR)) } } @@ -431,7 +809,8 @@ struct FileSystemPathResolver: Sendable { } -extension LimitedFileSystem: FileSystemChangeSource { +extension LimitedFileSystem: CompleteFileSystemChangeSource { + package var changeSignalsAreComplete: Bool { get async { await base.changeSignalsAreComplete } } public func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index b16562f..4d4a672 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -38,6 +38,8 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS private let stateLock = NSLock() private var mounts: [Mount] private var mountObservers: [UUID: AsyncStream<[Mount]>.Continuation] = [:] + private let syntheticDevice = UInt64.random(in: 1...UInt64.max) + private var syntheticInodes: [WorkspacePath: UInt64] = [:] private func withLock(_ body: () throws -> R) rethrows -> R { stateLock.lock() @@ -143,7 +145,7 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS } catch { guard Self.isMissing(error), path.isRoot || !syntheticChildMountNames(under: path, mounts: mounts).isEmpty else { throw error } - return Self.syntheticInfo(at: path) + return syntheticInfo(at: path) } } @@ -436,8 +438,15 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS return adjusted } - private static func syntheticInfo(at path: WorkspacePath) -> FileInfo { - FileInfo(path: path, kind: .directory, size: 0, permissions: .defaultDirectory, modificationDate: nil) + private func syntheticInfo(at path: WorkspacePath) -> FileInfo { + let inode = withLock { + if let existing = syntheticInodes[path] { return existing } + let next = UInt64(syntheticInodes.count) + 1 + syntheticInodes[path] = next + return next + } + return FileInfo(path: path, kind: .directory, size: 0, permissions: .defaultDirectory, + modificationDate: nil, identity: FileIdentity(device: syntheticDevice, inode: inode)) } private static func isMissing(_ error: Error) -> Bool { @@ -495,7 +504,15 @@ public final class MountedFileSystem: FileSystem, TraversingFileSystem, PreviewS } } -extension MountedFileSystem: FileSystemChangeSource { +extension MountedFileSystem: CompleteFileSystemChangeSource { + package var changeSignalsAreComplete: Bool { + get async { + let mounts = mountsSnapshot() + guard await base.changeSignalsAreComplete else { return false } + for mount in mounts where await !mount.fileSystem.changeSignalsAreComplete { return false } + return true + } + } public func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } @@ -516,8 +533,10 @@ extension MountedFileSystem: FileSystemChangeSource { subscriptions.removeAll() let sources = [Mount(mountPoint: .root, fileSystem: self.base)] + mounts for mount in sources { - guard let source = mount.fileSystem as? any FileSystemChangeSource, - let signals = try? await source.changeSignals() else { continue } + guard let source = mount.fileSystem as? any FileSystemChangeSource else { continue } + let signals: AsyncStream + do { signals = try await source.changeSignals() } + catch { continuation.finish(); return } subscriptions.append(Task { for await signal in signals where !Task.isCancelled { switch signal { @@ -534,6 +553,8 @@ extension MountedFileSystem: FileSystemChangeSource { if !visible.isEmpty { continuation.yield(.paths(visible)) } } } + // A failed child must not leave a live, silently incomplete stream. + if !Task.isCancelled { continuation.finish() } }) } // Reconcile changes during subscription setup and any new mount. diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index a87fb04..5ac1eb1 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -174,7 +174,8 @@ public final class SandboxFileSystem: FileSystem, TraversingFileSystem, PreviewS typealias SandboxFilesystem = SandboxFileSystem -extension SandboxFileSystem: FileSystemChangeSource { +extension SandboxFileSystem: CompleteFileSystemChangeSource { + package var changeSignalsAreComplete: Bool { get async { await (backing as any FileSystem).changeSignalsAreComplete } } public func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index 6b87164..b51ad15 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -299,7 +299,8 @@ public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, P typealias SecurityScopedFilesystem = SecurityScopedFileSystem -extension SecurityScopedFileSystem: FileSystemChangeSource { +extension SecurityScopedFileSystem: CompleteFileSystemChangeSource { + package var changeSignalsAreComplete: Bool { get async { await (backing as any FileSystem).changeSignalsAreComplete } } public func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 4c81684..751c70f 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -532,7 +532,10 @@ public final class AuthorizedFileSystem: FileSystem, PreviewSemanticsFileSystem, } -extension AuthorizedFileSystem: FileSystemChangeSource { +extension AuthorizedFileSystem: CompleteFileSystemChangeSource { + /// Signals for paths the authorizer hides are dropped, and decisions can change over time, so + /// consumers must not treat this stream as complete coverage of the base. + package var changeSignalsAreComplete: Bool { get async { false } } public func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } diff --git a/Tests/WorkspaceTests/LimitedCacheTests.swift b/Tests/WorkspaceTests/LimitedCacheTests.swift new file mode 100644 index 0000000..6e04adb --- /dev/null +++ b/Tests/WorkspaceTests/LimitedCacheTests.swift @@ -0,0 +1,568 @@ +import Foundation +import Testing +#if os(Linux) +import Glibc +#endif +@testable import Workspace + +@Suite("Limited accounting cache", .serialized) +struct LimitedCacheTests { + // Quota errors expose the projected totals at the public boundary. The independent + // oracle enumerates the real backend after each mutation, without resetting the cache. + private func checkTotals(_ limited: LimitedFileSystem, base: any FileSystem) async throws { + var bytes: UInt64 = 0 + var entries = 0 + var pending: [WorkspacePath] = [.root] + while let path = pending.popLast() { + let info = try await base.stat(path: path) + if !path.isRoot { entries += 1 } + if info.kind == .file { bytes += info.size } + if info.kind == .directory { + pending.append(contentsOf: try await base.listDirectory(path: path).map { path.appending($0.name) }) + } + } + await #expect(throws: FileSystemLimitError.totalBytes(attempted: bytes + 101, limit: 100)) { + try await limited.createFile(path: "/byte-probe", data: Data(count: 101)) + } + let deep = WorkspacePath(normalizing: (0..<17).map { "entry-probe-\($0)" }.joined(separator: "/")) + await #expect(throws: FileSystemLimitError.entryCount(attempted: entries + 17, limit: 16)) { + try await limited.createDirectory(path: deep, recursive: true) + } + } + + @Test(arguments: ["memory", "signals", "opaque aliases"]) + func mutationSequencesMatchFreshMetadata(backend: String) async throws { + let memory = InMemoryFileSystem() + let base: any FileSystem = backend == "memory" ? memory + : LimitedCacheBackend(memory: memory, opaqueAliases: backend == "opaque aliases") + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: memory) + try await fs.createDirectory(path: "/dir/sub", recursive: true) + try await checkTotals(fs, base: memory) + try await fs.createFile(path: "/dir/sub/file", data: Data(count: 3)) + try await checkTotals(fs, base: memory) + try await fs.writeFile(path: "/dir/sub/file", data: Data(count: 2), append: true) + try await checkTotals(fs, base: memory) + try await fs.createHardLink(path: "/alias", target: "/dir/sub/file") + try await checkTotals(fs, base: memory) + try await fs.writeFile(path: "/alias", data: Data(count: 1), append: true) + try await checkTotals(fs, base: memory) + try await fs.writeFile(path: "/alias", data: Data(count: 2), append: false) + try await checkTotals(fs, base: memory) + try await fs.createSymlink(path: "/sym", target: "/dir/sub/file") + try await checkTotals(fs, base: memory) + try await fs.writeFile(path: "/sym", data: Data(count: 1), append: true) + try await checkTotals(fs, base: memory) + try await fs.createSymlink(path: "/dangling", target: "/missing") + try await checkTotals(fs, base: memory) + try await fs.writeFile(path: "/dangling", data: Data(count: 4), append: false) + try await checkTotals(fs, base: memory) + try await fs.copy(from: "/dir", to: "/copy", recursive: true) + try await checkTotals(fs, base: memory) + try await fs.move(from: "/copy", to: "/moved") + try await checkTotals(fs, base: memory) + try await fs.remove(path: "/dir", recursive: true) + try await checkTotals(fs, base: memory) + try await fs.createHardLink(path: "/other", target: "/missing") + try await checkTotals(fs, base: memory) + try await fs.writeFile(path: "/missing", data: Data(count: 1), append: false) + try await checkTotals(fs, base: memory) + try await fs.remove(path: "/moved", recursive: true) + try await checkTotals(fs, base: memory) + try await fs.remove(path: "/does-not-exist", recursive: true) + try await checkTotals(fs, base: memory) + } + + @Test(arguments: [false, true]) + func warmMutationsDoNotEnumerateUnchangedSubtrees(mountedAliases: Bool) async throws { + let base = LimitedCacheBackend() + for i in 0..<200 { + try await base.memory.createDirectory(path: WorkspacePath("/dir-\(i)"), recursive: false) + try await base.memory.createFile(path: WorkspacePath("/dir-\(i)/file"), data: Data(count: 2)) + } + let wrapped: any FileSystem = mountedAliases ? MountedFileSystem(base: LimitedCacheBackend(), mounts: [ + .init(mountPoint: "/a", fileSystem: base), .init(mountPoint: "/b", fileSystem: base) + ]) : base + let fs = LimitedFileSystem(base: wrapped, limits: .init(maxTotalBytes: 10_000)) + let file: WorkspacePath = mountedAliases ? "/a/dir-0/file" : "/dir-0/file" + let new: WorkspacePath = mountedAliases ? "/a/dir-0/new" : "/dir-0/new" + try await fs.writeFile(path: file, data: Data(count: 3), append: false) + await base.clearReads() + try await fs.writeFile(path: file, data: Data(count: 1), append: true) + try await fs.createFile(path: new, data: Data(count: 1)) + try await fs.remove(path: new, recursive: false) + #expect(await base.listed.isEmpty) + } + + @Test + func externalScopesReconcileOnlyTheirCoverage() async throws { + let base = LimitedCacheBackend() + try await base.memory.createDirectory(path: "/untouched/deep", recursive: true) + try await base.memory.writeFile(path: "/untouched/deep/file", data: Data(count: 2), append: false) + try await base.memory.createDirectory(path: "/changed/old", recursive: true) + try await base.memory.writeFile(path: "/changed/file", data: Data(count: 3), append: false) + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: base.memory) + await base.clearReads() + try await base.memory.writeFile(path: "/changed/file", data: Data(count: 6), append: false) + await base.signal(.paths(["/changed/file": .entry])) + try await waitForBytes(8, fs: fs) + #expect(await base.listed.isEmpty) + try await checkTotals(fs, base: base.memory) + + await base.clearReads() + try await base.memory.remove(path: "/changed/old", recursive: true) + try await base.memory.createDirectory(path: "/changed/new/deep", recursive: true) + try await base.memory.writeFile(path: "/changed/new/deep/file", data: Data(count: 4), append: false) + await base.signal(.paths(["/changed": .shallow])) + try await waitForBytes(12, fs: fs) + #expect(await base.listed.contains("/changed/new/deep")) + #expect(!(await base.listed.contains("/untouched"))) + try await checkTotals(fs, base: base.memory) + + await base.clearReads() + try await base.memory.remove(path: "/changed", recursive: true) + try await base.memory.createSymlink(path: "/changed", target: "/untouched/deep") + await base.signal(.paths(["/changed": .entry])) + try await waitForBytes(2, fs: fs) + try await checkTotals(fs, base: base.memory) + #expect(!(await base.listed.contains("/untouched"))) + // Delayed events for the former directory's children must not follow its + // replacement symlink and charge the target file a second time. + try await base.memory.writeFile(path: "/untouched/deep/file", data: Data(count: 3), append: false) + await base.signal(.paths(["/changed/file": .entry, "/untouched/deep/file": .entry])) + try await waitForBytes(3, fs: fs) + try await checkTotals(fs, base: base.memory) + + try await base.memory.createDirectory(path: "/added/deep", recursive: true) + try await base.memory.writeFile(path: "/added/deep/file", data: Data(count: 5), append: false) + await base.signal(.paths(["/added": .recursive])) + try await waitForBytes(8, fs: fs) + try await checkTotals(fs, base: base.memory) + } + + @Test(arguments: ["all", "root"]) + func opaqueDirectoryAliasesInvalidateUncertainMutationEffects(missingIdentities: String) async throws { + let base = LimitedCacheBackend(opaqueAliases: missingIdentities == "all", opaqueRootIdentity: missingIdentities == "root") + try await base.memory.createDirectory(path: "/dir", recursive: false) + try await base.memory.createFile(path: "/dir/file", data: Data(count: 3)) + let mounted = missingIdentities == "root" + ? MountedFileSystem(base: base, mounts: [.init(mountPoint: "/a", fileSystem: base.memory)]) + : MountedFileSystem(base: LimitedCacheBackend(), mounts: [ + .init(mountPoint: "/a", fileSystem: base), .init(mountPoint: "/b", fileSystem: base) + ]) + let fs = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: mounted) + try await fs.createFile(path: "/a/new", data: Data(count: 2)) + try await checkTotals(fs, base: mounted) + try await fs.writeFile(path: "/a/dir/file", data: Data(count: 5), append: false) + try await checkTotals(fs, base: mounted) + try await fs.remove(path: "/a/new", recursive: false) + try await checkTotals(fs, base: mounted) + } + + @Test + func directoryAliasRemovalClearsChildrenWithoutFileIdentities() async throws { + let base = LimitedCacheBackend(opaqueFileIdentities: true) + try await base.memory.createDirectory(path: "/dir", recursive: false) + try await base.memory.createFile(path: "/dir/file", data: Data(count: 3)) + let mounted = MountedFileSystem(base: LimitedCacheBackend(), mounts: [ + .init(mountPoint: "/a", fileSystem: base), .init(mountPoint: "/b", fileSystem: base) + ]) + let fs = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: mounted) + try await base.memory.remove(path: "/dir", recursive: true) + await base.signal(.paths(["/dir": .entry])) + try await waitForBytes(0, fs: fs) + try await checkTotals(fs, base: mounted) + } + + @Test + func rescanTerminationAndSubscriptionFailureDiscardCachedUsage() async throws { + let base = LimitedCacheBackend() + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: base.memory) + try await base.memory.writeFile(path: "/external", data: Data(count: 8), append: false) + await base.signal(.rescan) + try await waitForBytes(8, fs: fs) + #expect(await base.listed.filter { $0.isRoot }.count >= 2) + await base.finish() + try await base.memory.writeFile(path: "/external", data: Data(count: 10), append: false) + try await waitForBytes(10, fs: fs) + try await base.memory.writeFile(path: "/external", data: Data(count: 12), append: false) + try await checkTotals(fs, base: base.memory) + + let unavailable = LimitedCacheBackend(subscriptionFails: true) + let fallback = LimitedFileSystem(base: unavailable, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fallback, base: unavailable.memory) + try await unavailable.memory.writeFile(path: "/external", data: Data(count: 7), append: false) + try await checkTotals(fallback, base: unavailable.memory) + } + + @Test + func revalidationClosesUndeliveredSignalGap() async throws { + let base = LimitedCacheBackend() + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: base.memory) + try await base.memory.writeFile(path: "/external", data: Data(count: 8), append: false) + // No signal has been delivered. The cached snapshot is permitted to remain old. + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 101, limit: 100)) { + try await fs.createFile(path: "/probe", data: Data(count: 101)) + } + try await fs.revalidate() + try await checkTotals(fs, base: base.memory) + } + + @Test + func partialMutationAndRefreshFailureRecoverOnNextPreflight() async throws { + let base = LimitedCacheBackend() + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: base.memory) + await base.failNextMutation() + await #expect(throws: CacheTestError.injected) { + try await fs.copy(from: "/", to: "/partial", recursive: true) + } + try await checkTotals(fs, base: base.memory) + await base.failNextRefresh() + try await fs.createFile(path: "/successful", data: Data(count: 5)) + #expect(try await base.memory.stat(path: "/successful").size == 5) + try await checkTotals(fs, base: base.memory) + } + + @Test + func signalDuringMutationSurvivesCacheCommitAndGateSerializesWriters() async throws { + let base = LimitedCacheBackend() + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 10)) + try await fs.revalidate() + let latch = CacheMutationLatch() + await base.pauseNextWrite(latch) + let first = Task { try await fs.writeFile(path: "/first", data: Data(count: 4), append: false) } + await latch.waitForEntry() + try await base.memory.writeFile(path: "/external", data: Data(count: 3), append: false) + await base.signal(.paths(["/external": .entry])) + let others = Task { + try await withThrowingTaskGroup(of: Bool.self) { group in + for i in 0..<20 { + group.addTask { + do { + try await fs.createFile(path: WorkspacePath("/other-\(i)"), data: Data(count: 4)) + return true + } catch is FileSystemLimitError { return false } + } + } + var successes = 0 + for try await success in group where success { successes += 1 } + return successes + } + } + await latch.release() + try await first.value + #expect(try await others.value <= 1) // Delivery may lag a preflight; the gate never overspends its snapshot. + // A delivered signal must survive the successful mutation's readback. + try await waitForBytes(try await base.memory.listDirectory(path: .root).reduce(0) { $0 + $1.info.size }, fs: fs, limit: 10) + } + + @Test + func memoryGenerationDetectsExternalChangesIncludingPartialFailureAndReset() async throws { + let memory = InMemoryFileSystem() + let fs = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: memory) + try await memory.writeFile(path: "/external", data: Data(count: 5), append: false) + try await checkTotals(fs, base: memory) + try await memory.createDirectory(path: "/dir", recursive: false) + try await memory.createFile(path: "/dir/file", data: Data(count: 2)) + try await memory.createHardLink(path: "/alias", target: "/dir/file") + try await checkTotals(fs, base: memory) + try await memory.writeFile(path: "/alias", data: Data(count: 1), append: true) + try await checkTotals(fs, base: memory) + try await memory.move(from: "/external", to: "/moved") + try await memory.copy(from: "/dir", to: "/copy", recursive: true) + try await memory.createSymlink(path: "/link", target: "/absent") + try await checkTotals(fs, base: memory) + try await memory.remove(path: "/copy", recursive: true) + await memory.reset() + try await checkTotals(fs, base: memory) + } + + @Test + func oversizedAccountingIsDiscardedBetweenRejectedPreflights() async throws { + let base = LimitedCacheBackend(fixedEntryCount: LimitedFileSystem.accountingCacheEntryLimit) + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100)) + try await fs.revalidate() + await base.clearReads() + for _ in 0..<2 { + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 101, limit: 100)) { + try await fs.createFile(path: "/probe", data: Data(count: 101)) + } + } + #expect(await base.listed.filter { $0.isRoot }.count == 2) + } + + @Test + func permissionsRefreshOnlyTheirSubtreeAndShallowReusesListingMetadata() async throws { + let base = LimitedCacheBackend() + try await base.memory.createDirectory(path: "/changed/sub", recursive: true) + try await base.memory.createDirectory(path: "/untouched", recursive: false) + try await base.memory.writeFile(path: "/changed/sub/file", data: Data(count: 3), append: false) + try await base.memory.writeFile(path: "/untouched/file", data: Data(count: 4), append: false) + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await checkTotals(fs, base: base.memory) + await base.clearReads() + try await fs.setPermissions(path: "/changed", permissions: .defaultDirectory) + try await checkTotals(fs, base: base.memory) + #expect(await base.listed.contains("/changed/sub")) + #expect(!(await base.listed.contains(.root))) + #expect(!(await base.listed.contains("/untouched"))) + await base.clearReads() + try await base.memory.writeFile(path: "/changed/new", data: Data(count: 2), append: false) + await base.signal(.paths(["/changed": .shallow])) + try await waitForBytes(9, fs: fs) + #expect(!(await base.statted.contains("/changed/new"))) + #expect(!(await base.statted.contains("/changed/sub"))) + try await checkTotals(fs, base: base.memory) + } + + @Test + func mountedSubscriptionFailureCannotRetainAccounting() async throws { + let unavailable = LimitedCacheBackend(subscriptionFails: true) + let mounted = MountedFileSystem(base: LimitedCacheBackend(), mounts: [ + .init(mountPoint: "/failed", fileSystem: unavailable) + ]) + let fs = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await fs.revalidate() + try await unavailable.memory.writeFile(path: "/external", data: Data(count: 8), append: false) + try await waitForBytes(8, fs: fs) + try await unavailable.memory.writeFile(path: "/external", data: Data(count: 12), append: false) + try await checkTotals(fs, base: mounted) + } + + @Test + func syntheticMountParentsKeepWarmAccounting() async throws { + let base = LimitedCacheBackend() + let mounted = MountedFileSystem(base: base, mounts: [ + .init(mountPoint: "/deep/m", fileSystem: LimitedCacheBackend()) + ]) + let fs = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await fs.writeFile(path: "/deep/m/a", data: Data(count: 3), append: false) + try await Task.sleep(for: .milliseconds(100)) // Initial topology rescan. + try await checkTotals(fs, base: mounted) + await base.clearReads() + try await fs.writeFile(path: "/deep/m/a", data: Data(count: 1), append: true) + #expect(await base.listed.isEmpty) + try await checkTotals(fs, base: mounted) + } + + #if os(macOS) || os(Linux) + @Test + func realLocalExternalSignalsConvergeToFreshWalk() async throws { + let root = try TestSupport.temporaryDirectory("LimitedNativeSignals") + defer { TestSupport.remove(root) } + let base = try LocalFileSystem(root: root) + // Require a working native watcher; a full-walk fallback would hide this regression. + let control = try await base.changeSignals() + let monitor = Task { for await _ in control {} } + defer { monitor.cancel() } + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) + try await fs.writeFile(path: "/a", data: Data(count: 3), append: false) + try await base.writeFile(path: "/dir/file", data: Data(count: 5), append: false) + try await waitForBytes(8, fs: fs) + try await checkTotals(fs, base: base) + try await base.createHardLink(path: "/alias", target: "/dir/file") + try await base.writeFile(path: "/alias", data: Data(count: 2), append: true) + try await waitForBytes(17, fs: fs) + try await checkTotals(fs, base: base) + try await base.move(from: "/dir", to: "/moved") + try await base.createSymlink(path: "/dangling", target: "/absent") + try await base.remove(path: "/a", recursive: false) + try await waitForBytes(14, fs: fs) + try await checkTotals(fs, base: base) + try await base.remove(path: "/moved", recursive: true) + try await waitForBytes(7, fs: fs) + try await checkTotals(fs, base: base) + #if os(Linux) + if geteuid() != 0 { + try await base.createDirectory(path: "/locked", recursive: false) + try await base.setPermissions(path: "/locked", permissions: POSIXPermissions(0)) + // Wait until registration has encountered the unwatchable child. + try await Task.sleep(for: .milliseconds(100)) + await #expect(throws: (any Error).self) { try await fs.revalidate() } + try await base.setPermissions(path: "/locked", permissions: .defaultDirectory) + try await base.writeFile(path: "/locked/file", data: Data(count: 2), append: false) + try await waitForBytes(9, fs: fs) + try await Task.sleep(for: .milliseconds(100)) + try await waitForBytes(9, fs: fs) + // After the permission event has been consumed, a second child write must + // have its own watch; a stale shallow invalidation cannot hide a gap. + try await base.writeFile(path: "/locked/file", data: Data(count: 3), append: false) + try await waitForBytes(10, fs: fs) + try await checkTotals(fs, base: base) + } + #endif + } + #endif + + private func waitForBytes(_ bytes: UInt64, fs: LimitedFileSystem, limit: UInt64 = 100) async throws { + let deadline = ContinuousClock.now + .seconds(3) + while true { + do { + try await fs.createFile(path: "/byte-probe", data: Data(count: Int(limit + 1))) + Issue.record("quota probe unexpectedly succeeded") + return + } catch let error as FileSystemLimitError { + if error == .totalBytes(attempted: bytes + limit + 1, limit: limit) { return } + if ContinuousClock.now >= deadline { Issue.record("unexpected usage: \(error), expected \(bytes)"); return } + } + await Task.yield() + } + } +} + +private enum CacheTestError: Error { case injected } + +// Own mutations do not emit events: cache correctness cannot depend on self-notifications. +// All content reads fail, so these tests also protect metadata-only accounting. +private actor LimitedCacheBackend: CompleteFileSystemChangeSource, PotentialPathResolvingFileSystem { + var changeSignalsAreComplete: Bool { true } + nonisolated let memory: InMemoryFileSystem + private let stream: AsyncStream + private let continuation: AsyncStream.Continuation + private let subscriptionFails: Bool + private let fixedEntryCount: Int + private let opaqueAliases: Bool + private let opaqueFileIdentities: Bool + private let opaqueRootIdentity: Bool + private var partialFailure = false + private var refreshFailure = false + private var failStat = false + private var latch: CacheMutationLatch? + private(set) var listed: [WorkspacePath] = [] + private(set) var statted: [WorkspacePath] = [] + init(memory: InMemoryFileSystem = InMemoryFileSystem(), fixedEntryCount: Int = 0, subscriptionFails: Bool = false, opaqueAliases: Bool = false, opaqueFileIdentities: Bool = false, opaqueRootIdentity: Bool = false) { + self.memory = memory + self.fixedEntryCount = fixedEntryCount + self.subscriptionFails = subscriptionFails + self.opaqueAliases = opaqueAliases + self.opaqueFileIdentities = opaqueFileIdentities + self.opaqueRootIdentity = opaqueRootIdentity + (stream, continuation) = AsyncStream.makeStream() + } + func signal(_ signal: FileSystemChangeSignal) { continuation.yield(signal) } + func finish() { continuation.finish() } + func clearReads() { listed.removeAll(); statted.removeAll() } + func failNextMutation() { partialFailure = true } + func failNextRefresh() { refreshFailure = true } + func pauseNextWrite(_ latch: CacheMutationLatch) { self.latch = latch } + func changeSignals() async throws -> AsyncStream { + if subscriptionFails { throw CacheTestError.injected } + return stream + } + func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } + func stat(path: WorkspacePath) async throws -> FileInfo { + statted.append(path) + if path.dirname.isRoot, path.basename.hasPrefix("fixed-"), + let index = Int(path.basename.dropFirst(6)), (0.. FileInfo { + var info = info + if opaqueAliases { info.identity = nil; info.linkCount = nil } + if opaqueFileIdentities, info.kind == .file { info.identity = nil } + if opaqueRootIdentity, info.path.isRoot { info.identity = nil } + return info + } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + listed.append(path) + var entries = try await memory.listDirectory(path: path).map { + DirectoryEntry(name: $0.name, info: metadata($0.info)) + } + if path.isRoot { + entries += (0.. FileInfo { + FileInfo(path: WorkspacePath(normalizing: "/fixed-\(index)"), kind: .file, size: 0, + permissions: .defaultFile, modificationDate: nil, + identity: FileIdentity(device: 0, inode: UInt64(index)), linkCount: 1) + } + func readFile(path: WorkspacePath) async throws -> Data { throw CacheTestError.injected } + func exists(path: WorkspacePath) async -> Bool { await memory.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await memory.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func createFile(path: WorkspacePath, data: Data) async throws { + try await memory.createFile(path: path, data: data) + if refreshFailure { refreshFailure = false; failStat = true } + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + if let latch { self.latch = nil; await latch.enter() } + try await memory.writeFile(path: path, data: data, append: append) + } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { + try await memory.createDirectory(path: path, recursive: recursive) + } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await memory.remove(path: path, recursive: recursive) } + func move(from: WorkspacePath, to: WorkspacePath) async throws { try await memory.move(from: from, to: to) } + func copy(from: WorkspacePath, to: WorkspacePath, recursive: Bool) async throws { + if partialFailure { + partialFailure = false + try await memory.createDirectory(path: to, recursive: true) + try await memory.createFile(path: to.appending("written-before-failure"), data: Data(count: 7)) + throw CacheTestError.injected + } + try await memory.copy(from: from, to: to, recursive: recursive) + } + func createSymlink(path: WorkspacePath, target: String) async throws { try await memory.createSymlink(path: path, target: target) } + func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { try await memory.createHardLink(path: path, target: target) } + func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { + try await memory.setPermissions(path: path, permissions: permissions) + } + func readSymlink(path: WorkspacePath) async throws -> String { try await memory.readSymlink(path: path) } + func capabilities() async -> FileSystemFeatures { await memory.capabilities() } + func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { try await memory.resolveRealPath(path: path) } + func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { try await memory.resolvePotentialPath(path: path) } +} + +private actor CacheMutationLatch { + private var entered = false + private var writer: CheckedContinuation? + private var observer: CheckedContinuation? + func enter() async { + entered = true + observer?.resume() + observer = nil + await withCheckedContinuation { writer = $0 } + } + func waitForEntry() async { if !entered { await withCheckedContinuation { observer = $0 } } } + func release() { writer?.resume(); writer = nil } +} + +extension LimitedCacheTests { + @Test + func `authorization filtering disables cached coverage so hidden external growth is counted`() async throws { + let root = try TestSupport.temporaryDirectory("LimitedAuthorizedCoverage") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/dir/sub", recursive: true) + try await local.writeFile(path: "/dir/sub/secret", data: Data(count: 1), append: false) + // A stat denial makes AuthorizedFileSystem drop that path's change signals. + let authorized = AuthorizedFileSystem(base: local, authorizer: PermissionAuthorizer { request in + request.operation == .stat && request.path == "/dir/sub/secret" ? .deny(message: nil) : .allow + }) + #expect(await authorized.changeSignalsAreComplete == false) + let limited = LimitedFileSystem(base: authorized, limits: .init(maxTotalBytes: 100)) + try await limited.writeFile(path: "/a", data: Data(count: 10), append: false) + + let handle = try FileHandle(forWritingTo: root.appendingPathComponent("dir/sub/secret")) + try handle.seekToEnd() + try handle.write(contentsOf: Data(count: 84)) + try handle.close() + + await #expect(throws: (any Error).self) { + try await limited.writeFile(path: "/b", data: Data(count: 10), append: false) + } + } +} diff --git a/Tests/WorkspaceTests/LimitedCoverageTests.swift b/Tests/WorkspaceTests/LimitedCoverageTests.swift new file mode 100644 index 0000000..ad8ff61 --- /dev/null +++ b/Tests/WorkspaceTests/LimitedCoverageTests.swift @@ -0,0 +1,27 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Limited observation coverage", .serialized) +struct LimitedCoverageTests { + @Test(arguments: ["memory base", "memory mount", "later memory mount"]) + func incompleteMountedCoverageSeesDirectWrites(configuration: String) async throws { + let root = try TestSupport.temporaryDirectory("LimitedCoverage") + defer { TestSupport.remove(root) } + let memory = InMemoryFileSystem() + let mounted = configuration == "memory base" ? MountedFileSystem(base: memory) + : MountedFileSystem(base: try LocalFileSystem(root: root)) + if configuration == "memory mount" { mounted.mount("/memory", fileSystem: memory) } + let fs = LimitedFileSystem(base: mounted, limits: .init(maxTotalBytes: 100)) + try await fs.writeFile(path: "/a", data: Data(count: 10), append: false) + // Let the initial topology rescan settle, so it cannot mask missing coverage. + try await Task.sleep(for: .milliseconds(300)) + if configuration == "later memory mount" { mounted.mount("/memory", fileSystem: memory) } + try await memory.writeFile(path: "/external", data: Data(count: 85), append: false) + try await Task.sleep(for: .milliseconds(1500)) + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 105, limit: 100)) { + try await fs.writeFile(path: "/b", data: Data(count: 10), append: false) + } + #expect(!(await mounted.exists(path: "/b"))) + } +} From ec2f4146aad9eabcfbb06781ad70c1e4c091709f Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 06:40:47 -0700 Subject: [PATCH 21/34] Make racy-metadata tests independent of scheduling The racy-window tests stamped files with the current second and relied on the next capture starting within two seconds, so they failed under load. They now use timestamps that are always inside the window and overwrite content in place, so the racy rule, not an identity change, is what forces the rehash. --- Tests/WorkspaceTests/ChangeSoundnessTests.swift | 3 ++- Tests/WorkspaceTests/PersistenceTests.swift | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Tests/WorkspaceTests/ChangeSoundnessTests.swift b/Tests/WorkspaceTests/ChangeSoundnessTests.swift index e53e579..e8b16c8 100644 --- a/Tests/WorkspaceTests/ChangeSoundnessTests.swift +++ b/Tests/WorkspaceTests/ChangeSoundnessTests.swift @@ -12,7 +12,8 @@ final class CoarseFileSystem: FileSystem, FileSystemChangeSource, Sendable { let timestamp: Date init(mode: Mode = .coarse, timestamp: Date? = nil) { self.mode = mode - self.timestamp = timestamp ?? (mode == .verified ? Date(timeIntervalSince1970: 1_000) : Date(timeIntervalSince1970: floor(Date().timeIntervalSince1970))) + // Coarse/racy cases must stay inside the capture window regardless of scheduling. + self.timestamp = timestamp ?? (mode == .verified ? Date(timeIntervalSince1970: 1_000) : .distantFuture) (stream, continuation) = AsyncStream.makeStream() } func signal(_ s: FileSystemChangeSignal) { continuation.yield(s) } diff --git a/Tests/WorkspaceTests/PersistenceTests.swift b/Tests/WorkspaceTests/PersistenceTests.swift index dd8e11b..43cada6 100644 --- a/Tests/WorkspaceTests/PersistenceTests.swift +++ b/Tests/WorkspaceTests/PersistenceTests.swift @@ -348,13 +348,18 @@ struct PersistenceTests { let workspace = Workspace(fileSystem: files, persistence: .directory(historyRoot)) try await local.writeFile(path: "/note", data: Data("first".utf8), append: false) try FileManager.default.setAttributes( - [.modificationDate: Date(timeIntervalSince1970: floor(Date().timeIntervalSince1970))], + [.modificationDate: Date().addingTimeInterval(60)], ofItemAtPath: filesRoot.appendingPathComponent("note").path ) let mtime = try #require(try await local.stat(path: "/note").modificationDate) + let identity = try #require(try await local.stat(path: "/note").identity) let before = try await workspace.createCheckpoint() - try await local.writeFile(path: "/note", data: Data("other".utf8), append: false) + // Keep the inode and size unchanged so only the racy window forces a read. + let handle = try FileHandle(forWritingTo: filesRoot.appendingPathComponent("note")) + try handle.write(contentsOf: Data("other".utf8)) + try handle.close() try FileManager.default.setAttributes([.modificationDate: mtime], ofItemAtPath: filesRoot.appendingPathComponent("note").path) + #expect(try await local.stat(path: "/note").identity == identity) #expect(try await local.stat(path: "/note").modificationDate == mtime) await files.resetReadCount() let after = try await workspace.createCheckpoint() From 94ef94b4d33b5ab867782a77d8e0e7b26a5490e8 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 06:40:47 -0700 Subject: [PATCH 22/34] Validate the benchmark fixture before each scenario If the shared fixture's file count or total bytes change during a run, the harness now stops with an error naming the last scenario that ran, instead of letting later scenarios fail on a partial tree. --- Benchmarks/WorkspaceBenchmarks/CLI.swift | 8 ++++++ Benchmarks/WorkspaceBenchmarks/Fixture.swift | 30 ++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/Benchmarks/WorkspaceBenchmarks/CLI.swift b/Benchmarks/WorkspaceBenchmarks/CLI.swift index 7cee669..5c19ccd 100644 --- a/Benchmarks/WorkspaceBenchmarks/CLI.swift +++ b/Benchmarks/WorkspaceBenchmarks/CLI.swift @@ -208,6 +208,14 @@ struct WorkspaceBenchmarks { ) for (index, scenario) in scenarios.enumerated() { try TemporaryCleanup.shared.checkInterrupt() + do { + try fixture.validateCounts(at: directory) + } catch { + try TemporaryCleanup.shared.checkInterrupt() + let last = report.results.last?.scenario.id ?? "none (before first scenario)" + throw BenchmarkError.message( + "fixture modified during run: \(error); last scenario: \(last); before scenario: \(scenario.id)") + } let resultURL = temporary.appendingPathComponent("result\(index).json") let child = Process() guard let executable = Bundle.main.executableURL else { diff --git a/Benchmarks/WorkspaceBenchmarks/Fixture.swift b/Benchmarks/WorkspaceBenchmarks/Fixture.swift index dbda5cc..b8028d3 100644 --- a/Benchmarks/WorkspaceBenchmarks/Fixture.swift +++ b/Benchmarks/WorkspaceBenchmarks/Fixture.swift @@ -165,6 +165,36 @@ struct Fixture: Codable, Sendable { try JSONEncoder().encode(fixture).write(to: manifest, options: .atomic) return fixture } + /// Metadata-only guard between children; full content validation happens on initial reuse. + func validateCounts(at directory: URL) throws { + let manager = FileManager.default + let tree = directory.appendingPathComponent("tree") + guard (try manager.attributesOfItem(atPath: tree.path))[.type] as? FileAttributeType == .typeDirectory else { + throw BenchmarkError.message("missing fixture tree: \(tree.path)") + } + var enumerationError: Error? + guard let enumerator = manager.enumerator(at: tree, + includingPropertiesForKeys: [.isRegularFileKey, .fileSizeKey], + errorHandler: { _, error in enumerationError = error; return false }) else { + throw BenchmarkError.message("cannot enumerate fixture tree: \(tree.path)") + } + var actualFiles = 0 + var actualBytes: Int64 = 0 + for case let url as URL in enumerator { + try TemporaryCleanup.shared.checkInterrupt() + let values = try url.resourceValues(forKeys: [.isRegularFileKey, .fileSizeKey]) + if values.isRegularFile == true { + actualFiles += 1 + actualBytes += Int64(values.fileSize ?? 0) + } + } + if let enumerationError { throw enumerationError } + guard actualFiles == fileCount, actualBytes == bytes else { + throw BenchmarkError.message( + "expected \(fileCount) regular files / \(bytes) bytes, found \(actualFiles) regular files / \(actualBytes) bytes") + } + } + static func loadMemory(from tree: URL) async throws -> InMemoryFileSystem { let local = try LocalFileSystem(root: tree) let memory = InMemoryFileSystem() From 4bdda03f313d42983da0269f64c0a5a8d1193d17 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 06:44:15 -0700 Subject: [PATCH 23/34] Tolerate device renumbering when reusing persisted identities FileIdentity includes the device number, which can change across reboots or remounts for external volumes, container overlays, NFS and some btrfs setups. A device-only change made every persisted identity mismatch, so the next checkpoint rewrote every tree and re-read every file, and restore, diff and transactions lost their fast path. When a persisted or cached identity differs from the current one only in device number, reuse and change detection now treat identity as unknown and fall back to size, mtime and the racy-window rule. A real replacement (different inode) still forces a read. Hard-link accounting and hashing keep exact device-and-inode identity. --- Sources/Workspace/CheckpointStore.swift | 9 +- Sources/Workspace/FS/FileSystem.swift | 14 +- Sources/Workspace/MetadataIndex.swift | 6 +- Sources/Workspace/Workspace.swift | 3 +- .../WorkspaceTests/DeviceIdentityTests.swift | 129 ++++++++++++++++++ 5 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 Tests/WorkspaceTests/DeviceIdentityTests.swift diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index e85c0e4..de17cfa 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -465,7 +465,8 @@ actor FileCheckpointStore: CheckpointStore { static func == (lhs: Self, rhs: Self) -> Bool { lhs.kind == rhs.kind && lhs.path == rhs.path && lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && - lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && lhs.identity == rhs.identity && + lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && + FileIdentity.matchesForMetadataReuse(lhs.identity, rhs.identity) && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && lhs.children == rhs.children && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } @@ -568,7 +569,8 @@ actor FileCheckpointStore: CheckpointStore { static func == (lhs: Self, rhs: Self) -> Bool { lhs.name.utf8.elementsEqual(rhs.name.utf8) && lhs.kind == rhs.kind && lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && - lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && lhs.identity == rhs.identity && + lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && + FileIdentity.matchesForMetadataReuse(lhs.identity, rhs.identity) && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } @@ -869,7 +871,8 @@ actor FileCheckpointStore: CheckpointStore { modificationDate.addingTimeInterval(2) < previousCaptureStartedAt, previous?.kind == .file, previous?.contentSize == info.size, - (previous?.identity == nil || info.identity == nil || previous?.identity == info.identity), + (previous?.identity == nil || info.identity == nil + || FileIdentity.matchesForMetadataReuse(previous?.identity, info.identity)), previous?.modificationDate != nil, previous?.modificationDate == info.modificationDate, let hash = previous?.contentHash, diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index 7ea3edc..6eef2f1 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -112,9 +112,9 @@ public struct FileInfo: Sendable, Codable { } /// Opaque identity of a filesystem entry, shared by aliases across backend instances. -/// Device and inode values must distinguish different entries across instances and remain -/// stable during an entry's lifetime; replacements receive a new identity. Independent -/// in-memory backends need distinct device tokens; disk backends can use st_dev and st_ino. +/// Device and inode values must distinguish live entries across backend instances; +/// replacements receive a new identity. Device numbers may change across remounts. +/// Independent in-memory backends need distinct device tokens; disk backends can use st_dev and st_ino. public struct FileIdentity: Hashable, Sendable, Codable { private let device: UInt64 private let inode: UInt64 @@ -123,6 +123,14 @@ public struct FileIdentity: Hashable, Sendable, Codable { self.device = device self.inode = inode } + + /// Only for persisted/cached metadata comparisons, never live alias detection. + /// Equal inodes with different devices may reflect a remount: identity is then + /// unknown, leaving size, mtime and the racy window to decide content reuse. + /// Missing identities compare equal only to each other here. + static func matchesForMetadataReuse(_ lhs: Self?, _ rhs: Self?) -> Bool { + lhs?.inode == rhs?.inode + } } /// A named child entry returned from a directory listing. diff --git a/Sources/Workspace/MetadataIndex.swift b/Sources/Workspace/MetadataIndex.swift index fcb7dad..343817d 100644 --- a/Sources/Workspace/MetadataIndex.swift +++ b/Sources/Workspace/MetadataIndex.swift @@ -9,7 +9,8 @@ struct MetadataIndex: Sendable { static func == (lhs: Self, rhs: Self) -> Bool { lhs.info.kind == rhs.info.kind && lhs.info.size == rhs.info.size && lhs.info.modificationDate == rhs.info.modificationDate - && lhs.info.identity == rhs.info.identity && lhs.info.permissions == rhs.info.permissions + && FileIdentity.matchesForMetadataReuse(lhs.info.identity, rhs.info.identity) + && lhs.info.permissions == rhs.info.permissions && lhs.target.map(PathName.init) == rhs.target.map(PathName.init) } @@ -17,7 +18,8 @@ struct MetadataIndex: Sendable { info.kind == .file && old.info.kind == .file && old.info.modificationDate.map { exactTimestamps || $0.addingTimeInterval(2) < capturedAt } == true && info.size == old.info.size && info.modificationDate == old.info.modificationDate - && (info.identity == nil || old.info.identity == nil || info.identity == old.info.identity) + && (info.identity == nil || old.info.identity == nil + || FileIdentity.matchesForMetadataReuse(info.identity, old.info.identity)) } } var entries: [WorkspacePath: Entry] = [:] diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 04fce86..85d615a 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -11,7 +11,8 @@ public actor Workspace { /// Controls content verification for transactions, checkpoint comparisons/restores, and directory checkpoint capture. public enum CheckpointCapturePolicy: Sendable, Equatable { /// Reuse matching size and mtime when mtime is over two seconds older than the prior - /// capture, comparing identity when both entries provide it. Exact-timestamp backends + /// capture, comparing identity when both entries provide it. Device-only changes with + /// equal inodes are treated as unknown identity. Exact-timestamp backends /// skip the racy window. Missing mtimes always require reading contents. case trustMetadata /// Read and verify every file, including changes that preserve older timestamps. diff --git a/Tests/WorkspaceTests/DeviceIdentityTests.swift b/Tests/WorkspaceTests/DeviceIdentityTests.swift new file mode 100644 index 0000000..a3be19e --- /dev/null +++ b/Tests/WorkspaceTests/DeviceIdentityTests.swift @@ -0,0 +1,129 @@ +import Foundation +import Testing +@testable import Workspace + +struct DeviceIdentityTests { + @Test(arguments: ["unchanged", "size", "mtime", "racy", "verified", "replacement"]) + func `device renumbering preserves capture reuse without bypassing content guards`(change: String) async throws { + let root = try TestSupport.temporaryDirectory("DeviceCapture") + defer { TestSupport.remove(root) } + let fs = RenumberedDeviceFileSystem(recent: change == "racy") + try await fs.createDirectory(path: "/dir", recursive: true) + try await fs.writeFile(path: "/dir/file", data: Data("old".utf8), append: false) + try await fs.base.createSymlink(path: "/link", target: "dir/file") + let policy: Workspace.CheckpointCapturePolicy = change == "verified" ? .verifyContents : .trustMetadata + let id = UUID() + let first = try await FileCheckpointStore(rootDirectory: root, checkpointCapture: policy).captureRevision( + from: fs, draft: .init(workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + let treesURL = root.appendingPathComponent(id.uuidString).appendingPathComponent("trees") + let beforeTrees = try treeArtifacts(treesURL) + await fs.renumber(replacingFile: change == "replacement") + if change != "unchanged" { + try await fs.writeFile(path: "/dir/file", data: Data((change == "size" ? "longer" : "new").utf8), append: false) + } + if change == "mtime" { await fs.advanceTimestamp() } + await fs.resetReads() + // Reopen so the comparison uses persisted identities, including nested trees. + let reopened = FileCheckpointStore(rootDirectory: root, checkpointCapture: policy) + let second = try await reopened.captureRevision(from: fs, draft: .init( + workspaceID: id, label: nil, preferredParentID: first.id, origin: .manual)) + #expect(await fs.reads == (change == "unchanged" ? [] : ["/dir/file"])) + #expect(second.summary.touchedPaths == (change == "unchanged" ? [] : ["/dir/file"])) + let expected = change == "unchanged" ? "old" : change == "size" ? "longer" : "new" + #expect(try await reopened.readSnapshotFile(id: second.snapshotId, workspaceId: id, + path: "/dir/file", offset: 0, length: nil) == Data(expected.utf8)) + if change == "unchanged" { #expect(try treeArtifacts(treesURL) == beforeTrees) } + } + + @Test(arguments: [false, true], [false, true]) + func `device renumbering retains diff and restore fast paths and the racy window`(disk: Bool, recent: Bool) async throws { + let root = try TestSupport.temporaryDirectory("DeviceComparison") + defer { TestSupport.remove(root) } + let fs = RenumberedDeviceFileSystem(recent: recent) + try await fs.createDirectory(path: "/dir", recursive: true) + try await fs.writeFile(path: "/dir/file", data: Data("old".utf8), append: false) + var workspace = Workspace(fileSystem: fs, persistence: disk ? .directory(root) : .memory) + let checkpoint = try await workspace.createCheckpoint() + if disk { + workspace = Workspace(workspaceID: workspace.workspaceID, fileSystem: fs, persistence: .directory(root)) + } + await fs.renumber() + await fs.resetReads() + #expect(try await workspace.diff(from: .checkpoint(checkpoint.id), to: .current).isEmpty) + #expect(await fs.reads == (recent ? ["/dir/file"] : [])) + await fs.resetReads() + _ = try await workspace.restore(to: checkpoint) + #expect(await fs.reads == (recent ? ["/dir/file"] : [])) + #expect(try await workspace.history().isEmpty) + } + + @Test(arguments: [false, true], [false, true]) + func `transactions tolerate device renumbering while verifying racy contents`(disk: Bool, recent: Bool) async throws { + let root = try TestSupport.temporaryDirectory("DeviceTransaction") + defer { TestSupport.remove(root) } + let fs = RenumberedDeviceFileSystem(recent: recent) + try await fs.createDirectory(path: "/dir", recursive: true) + try await fs.writeFile(path: "/dir/file", data: Data("old".utf8), append: false) + let workspace = Workspace(fileSystem: fs, persistence: disk ? .directory(root) : .memory) + // Keep the cold directory base alive so both storage modes have cached bytes. + let first = try await workspace.beginTransaction() + await fs.renumber() + if recent { + try await fs.writeFile(path: "/dir/file", data: Data("new".utf8), append: false) + } + await fs.resetReads() + let second = try await workspace.beginTransaction() + #expect(await fs.reads == (recent ? ["/dir/file"] : [])) + #expect(try await second.readText("/dir/file") == (recent ? "new" : "old")) + try await second.discard() + try await first.discard() + } + + private func treeArtifacts(_ directory: URL) throws -> [String: Data] { + try Dictionary(uniqueKeysWithValues: FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: nil).map { ($0.lastPathComponent, try Data(contentsOf: $0)) }) + } +} + +/// Models remounts and in-place writes independently of the underlying memory backend. +private actor RenumberedDeviceFileSystem: FileSystem { + let base = InMemoryFileSystem() + private var device: UInt64 = 1 + private var fileInode: UInt64 = 3 + private var timestamp: Date + var reads: [WorkspacePath] = [] + init(recent: Bool) { timestamp = recent ? .distantFuture : Date(timeIntervalSince1970: 1_000) } + func renumber(replacingFile: Bool = false) { + device += 1 + if replacingFile { fileInode += 1 } + } + func advanceTimestamp() { timestamp.addTimeInterval(10) } + func resetReads() { reads = [] } + private func metadata(_ info: FileInfo) -> FileInfo { + var info = info + let inode: UInt64 = info.path.isRoot ? 1 : info.kind == .directory ? 2 : info.kind == .file ? fileInode : 10 + info.identity = FileIdentity(device: device, inode: inode) + info.modificationDate = timestamp + return info + } + func stat(path: WorkspacePath) async throws -> FileInfo { metadata(try await base.stat(path: path)) } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { + try await base.listDirectory(path: path).map { .init(name: $0.name, info: metadata($0.info)) } + } + func readFile(path: WorkspacePath) async throws -> Data { + reads.append(path) + return try await base.readFile(path: path) + } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await base.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await base.writeFile(path: path, data: data, append: append) } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await base.remove(path: path, recursive: recursive) } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) + } +} From 764e01832d4a43fd4e7c54cf028f4654f90fffef Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 09:08:05 -0700 Subject: [PATCH 24/34] Drop backward-compatibility code Nothing has shipped, so earlier on-disk formats, serialized shapes and API spellings are no longer supported. The checkpoint store has a single format marker ({"format":"workspace-history","version":1}); any other or missing marker on an existing history directory is an error that asks for the directory to be deleted. Format migration and its nested locks, legacy inline-children manifests, persisted .missing children, the fail-closed treeRoot key, integer sequence sidecars and silent field defaults are gone; missing required fields now raise storageCorrupted. Search request and result types use synthesized Codable. The Void FileSystemChangeSource.changes() stream and Workspace.watchChanges(at:recursive:) are removed in favour of changeSignals() and events(_:) with a filter, along with several internal aliases, the v1 store fixture and compatibility-only tests. --- .../CountingFileSystem.swift | 13 - Package.swift | 3 +- README.md | 39 +-- Sources/Workspace/Checkpoint.swift | 7 - .../Workspace/CheckpointStore+Revision.swift | 12 +- Sources/Workspace/CheckpointStore.swift | 246 ++++++------------ Sources/Workspace/FS/FileSystem.swift | 19 -- .../Workspace/FS/FileSystemTraversal.swift | 4 +- Sources/Workspace/FS/LimitedFileSystem.swift | 6 +- Sources/Workspace/FS/LocalFileSystem.swift | 11 - Sources/Workspace/FS/MountedFileSystem.swift | 4 - Sources/Workspace/FS/SandboxFileSystem.swift | 7 - .../FS/SecurityScopedFileSystem.swift | 6 - .../Workspace/Support/FileChangeStreams.swift | 2 +- Sources/Workspace/Support/Permissions.swift | 4 - Sources/Workspace/TransactionTree.swift | 4 +- Sources/Workspace/Workspace+Checkpoints.swift | 2 +- .../Workspace+TransactionStorage.swift | 4 +- Sources/Workspace/WorkspaceEvent.swift | 8 - Sources/Workspace/WorkspaceRevision.swift | 10 +- Sources/Workspace/WorkspaceSearch.swift | 51 +--- Sources/Workspace/WorkspaceToolBudget.swift | 4 +- Tests/WorkspaceTests/ChangeScalingTests.swift | 1 - .../WorkspaceTests/ChangeSoundnessTests.swift | 2 - .../CheckpointFormatTests.swift | 111 +------- .../WorkspaceTests/CheckpointStoreTests.swift | 37 ++- Tests/WorkspaceTests/EditPathTests.swift | 2 +- .../FileSystemRegressionTests.swift | 10 - Tests/WorkspaceTests/FilesystemTests.swift | 26 +- Tests/WorkspaceTests/Fixtures/README.md | 7 - ...00-0000-0000-0000-000000000001.format.lock | 0 ...cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 | 1 - ...d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 | 1 - ...aff9b70ad0b84a09286df071b9159796462b635ae2 | 1 - .../669D476B-6EFC-46E0-A1FB-8155C3DA8121.json | 33 --- .../8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json | 28 -- .../format.json | 3 - .../lifecycle.lock | 0 .../mutations.jsonl | 2 - .../mutations.lock | 0 .../mutations.sequence.json | 4 - .../5D7DC62F-9749-41E4-8BCE-90306A25C30C.json | 65 ----- .../EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json | 65 ----- Tests/WorkspaceTests/LimitedCacheTests.swift | 1 - .../WorkspaceTests/ReviewRevisionTests.swift | 2 +- .../SearchEquivalenceTests.swift | 2 +- .../UnicodePathRegressionTests.swift | 2 +- Tests/WorkspaceTests/WorkspaceAPITests.swift | 6 +- 48 files changed, 177 insertions(+), 701 deletions(-) delete mode 100644 Tests/WorkspaceTests/Fixtures/README.md delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/.10000000-0000-0000-0000-000000000001.format.lock delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/lifecycle.lock delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.lock delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json delete mode 100644 Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json diff --git a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift index 734c180..0708ee4 100644 --- a/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift +++ b/Benchmarks/WorkspaceBenchmarks/CountingFileSystem.swift @@ -25,12 +25,6 @@ class CountingFileSystem: CompleteFileSystemChangeSource, MutationGenerationFile } return try await source.changeSignals() } - func changes() async throws -> AsyncStream { - guard let source = base as? any FileSystemChangeSource else { - throw BenchmarkError.message("backend does not provide filesystem notifications") - } - return try await source.changes() - } let base: any FileSystem private let lock = NSLock() private var counts = IOCounts() @@ -194,13 +188,6 @@ final class ObservedCountingFileSystem: CountingFileSystem, } return try await source.changeSignals() } - override func changes() async throws -> AsyncStream { - call("changes") - guard let source = base as? any FileSystemChangeSource else { - throw BenchmarkError.message("backend does not provide filesystem notifications") - } - return try await source.changes() - } } #endif diff --git a/Package.swift b/Package.swift index 47843ba..51d4525 100644 --- a/Package.swift +++ b/Package.swift @@ -29,8 +29,7 @@ let package = Package( ), .testTarget( name: "WorkspaceTests", - dependencies: ["Workspace"], - resources: [.copy("Fixtures")] + dependencies: ["Workspace"] ), ] ) diff --git a/README.md b/README.md index e954bc7..f09fbde 100644 --- a/README.md +++ b/README.md @@ -181,20 +181,11 @@ Compaction rebases retained checkpoints to their nearest retained ancestor and r Directory-backed stores serialize checkpoint commits, revision reads, and compaction with a persistent lifecycle lock. Mutation history is independent and is not pruned by checkpoint retention. -New directory stores use format version 2. Preparing an existing v1 store upgrades -`format.json` under the format, lifecycle, and mutation locks before publishing any new-format -checkpoint. Existing recursive manifests and loose blobs remain readable; migration does not -rewrite checkpoint history. The first new checkpoint over a v1 parent creates its directory -objects; historical v1 reads keep their original full-manifest decoding cost. New v1 operations -reject the v2 format marker. Schema-3 roots use the required `treeRoot` key instead of legacy -`root`, so an old operation admitted before migration fails decoding a new root: it cannot read -it as an empty tree, capture against it, or sweep its blobs during compaction. An admitted old -operation that only accesses legacy roots (or appends mutations) can still complete; migration -cannot cancel old code already running. Stop old processes before upgrading to avoid mixed-version -operation failures. -Do not downgrade the format marker. - -New snapshots use manifest schema 3: a small snapshot root points to SHA-256-addressed directory +Directory stores use the marker `{"format":"workspace-history","version":1}` in `format.json`. +An existing store directory must have that marker; a missing, malformed, or unsupported marker is an error instructing the +caller to delete the history directory. Store creation is serialized across instances and processes. + +A small snapshot root points to SHA-256-addressed directory objects in `trees/`. Each object contains byte-sorted immediate child names (never absolute paths), with child directories referenced by hash. An incremental capture writes only changed directories and their ancestors; unchanged objects remain shared. File reads decode only directories on the requested path. @@ -228,20 +219,20 @@ that honors these synchronization requests. If `F_FULLFSYNC` fails other than in publication retains the completed file and directory `fsync`; power-loss protection is best-effort on such volumes. Other POSIX platforms use file and directory `fsync`. Failed captures can leave unreachable objects, but publish no checkpoint referencing incomplete content. Compaction -sweeps both formats, rewrites partially live packs, durably replaces affected snapshot roots, +sweeps unreachable objects, rewrites partially live packs, durably replaces affected snapshot roots, and only then removes old objects. Interrupted compaction can be resumed; dry-run blob counts and byte totals include reclaimable pack entries and incomplete orphan packs. All writers preserve the mutation sequence high-water mark and retraction epoch in -`mutations.sequence.json`; legacy bare-integer sidecars remain readable. Retractions and failed +`mutations.sequence.json`. Retractions and failed append reservations can leave sequence gaps. History refreshes invalidate cached records when another instance retracts or prunes mutations. Directory-backed checkpoint capture also reuses existing CAS blobs when a file's size and available modification timestamp are unchanged and the timestamp is more than two seconds older than the previous capture's start. Entries within that timestamp-granularity window are rehashed. It still -walks directory metadata for correctness. Filesystems without modification timestamps, and older -manifests without a capture timestamp, safely fall back to reading those files. +walks directory metadata for correctness. Filesystems without modification timestamps and +materialized snapshots without capture hints require content reads. For tools that preserve older timestamps while changing content, pass `checkpointCapture: .verifyContents` to `Workspace` to read and hash every file at each checkpoint. @@ -251,7 +242,7 @@ network filesystems whose clock may differ. This policy affects only `.directory ## Transactions -Transactions replace separate branch and merge APIs: +Begin an isolated transaction: ```swift let transaction = try await workspace.beginTransaction(label: "agent draft") @@ -315,7 +306,7 @@ file during directory checkpoint capture. Checkpoint-to-current diffs and `restore(to:)` use one bulk metadata walk and compare against the checkpoint's persisted capture hints and revision hashes. They read only suspect files (changed metadata, missing hints, or timestamps inside the same two-second window). Identity is compared -when both captures provide it; exact-timestamp backends skip the window. Older revisions without +when both captures provide it; exact-timestamp backends skip the window. Revisions without capture hints conservatively require content reads. Memory persistence retains hints for only the four most recently captured or restored revisions; older revisions also fall back to content reads. As with transactions, writes that preserve @@ -350,7 +341,7 @@ let mutations = try await workspace.history() Change events, returned edit results, and mutation history all carry the same `ChangeSet` representation. Checkpoint events share the same stream. `LocalFileSystem` also pushes out-of-band filesystem invalidations through FSEvents on macOS and -inotify on Linux. `events()` and the path-focused `watchChanges(at:recursive:)` turn those +inotify on Linux. `events(_:)`, with an optional path filter, turns those invalidations into structured `ChangeSet` values. External changes remain unrecorded in mutation history, matching the direct-filesystem contract above. Directory-backed checkpoint stores use the same push model for cross-instance checkpoint events; there is no periodic polling loop. @@ -599,12 +590,6 @@ an apply call to retrieve omitted detail. Use `workspace_read` to inspect omitte a checkpoint before applying and diff from it. If response encoding fails after commit, apply returns a minimal success envelope with empty `changes`, full statistics, and `truncated: true`. -Beta compatibility: glob tool results now use an object instead of a bare array. Apply and diff -retain their existing keys and add budget/paging metadata; large results can contain less detail. -Direct Workspace method signatures are unchanged. Public adapter budget constants and -`TextDiffOmission.Reason.outputBudgetExceeded` are additive; exhaustive switches on the reason -must handle the new case. - ## Testing ```bash diff --git a/Sources/Workspace/Checkpoint.swift b/Sources/Workspace/Checkpoint.swift index 76fcc96..cdab063 100644 --- a/Sources/Workspace/Checkpoint.swift +++ b/Sources/Workspace/Checkpoint.swift @@ -23,13 +23,6 @@ public struct Checkpoint: Sendable, Codable, Equatable { var mutationCursor: Int var snapshotId: UUID - var workspaceId: UUID { workspaceID } - var parentCheckpointId: UUID? { parentID } - var rollbackSourceCheckpointId: UUID? { - if case let .rollback(from) = origin { return from } - return nil - } - var provenanceCheckpointID: UUID? { switch origin { case let .rollback(from): from diff --git a/Sources/Workspace/CheckpointStore+Revision.swift b/Sources/Workspace/CheckpointStore+Revision.swift index cb15a3d..5520497 100644 --- a/Sources/Workspace/CheckpointStore+Revision.swift +++ b/Sources/Workspace/CheckpointStore+Revision.swift @@ -37,7 +37,7 @@ struct CaptureMetadataCache: Sendable { } extension CheckpointStore { - // Conservative compatibility for decorators and stores without capture hints. + // Content verification for decorators and stores without capture hints. func loadCapturedRevision(id: UUID, workspaceId: UUID) async throws -> CapturedRevision? { guard let index = try await loadRevisionIndex(id: id, workspaceId: workspaceId) else { return nil } return CapturedRevision(index: index, metadata: MetadataIndex()) @@ -94,9 +94,9 @@ extension FileCheckpointStore { return try await withSharedLock(at: lifecycleLockURL(workspaceId: workspaceId)) { guard let manifest = try loadManifestUnlocked(id: id, workspaceId: workspaceId) else { return nil } var metadata = MetadataIndex() - // Legacy materialized snapshots have no trustworthy capture timestamp. + // Materialized snapshots have no trustworthy capture timestamp. metadata.capturedAt = manifest.captureStartedAt ?? .distantPast - metadata.exactTimestamps = manifest.exactContentTimestamps ?? false + metadata.exactTimestamps = manifest.exactContentTimestamps func visit(_ node: SnapshotManifest.Node) throws -> RevisionIndex.Entry { let entry = try revisionEntry(from: node, workspaceId: workspaceId, depth: 0) if let kind = entry.kind { @@ -134,7 +134,6 @@ extension FileCheckpointStore { } return manifest } - try requireCurrentFormatUnlocked(workspaceId: workspaceId) let batch = try makeBlobBatch(workspaceId: workspaceId) defer { batch.close() } func apply(_ patch: Snapshot.Entry, to old: SnapshotManifest.Node?) throws -> SnapshotManifest.Node { @@ -157,8 +156,7 @@ extension FileCheckpointStore { node.treeHash = old.treeHash return node } - node.children = updated - return try persistDirectory(node, workspaceId: workspaceId) + return try persistDirectory(node, children: updated, workspaceId: workspaceId) } var node = try writeManifestNode(patch, workspaceId: workspaceId, batch: batch, reusing: old) node.modificationDate = metadata.entries[patch.path]?.info.modificationDate @@ -167,7 +165,7 @@ extension FileCheckpointStore { } let root = try apply(patch.entry, to: source.root) try finish(batch, workspaceId: workspaceId) - let manifest = SnapshotManifest(version: 3, id: UUID(), rootPath: source.rootPath, + let manifest = SnapshotManifest(id: UUID(), rootPath: source.rootPath, root: root, captureStartedAt: metadata.capturedAt, exactContentTimestamps: metadata.exactTimestamps) let summary: ChangeSet.Summary diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index de17cfa..f73b93a 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -441,9 +441,7 @@ actor FileCheckpointStore: CheckpointStore { } } - /// Schemas 1/2 inline the recursive tree and reference loose blobs. Schema 3 - /// stores a small root, content-addressed directory references, and optional pack ranges. - /// The outer store format is version 2, fencing writers that only understand inline trees. + /// A small root with content-addressed directory references and optional pack ranges. struct SnapshotManifest: Codable, Sendable { struct Node: Codable, Sendable, Equatable { enum Kind: String, Codable { @@ -452,13 +450,12 @@ actor FileCheckpointStore: CheckpointStore { var kind: Kind var path: WorkspacePath - var permissions: POSIXPermissions? + var permissions: POSIXPermissions = .defaultFile var contentHash: String? var contentSize: UInt64? = nil var modificationDate: Date? = nil var identity: FileIdentity? = nil var symlinkTarget: String? - var children: [Node]? var treeHash: String? = nil var packed: PackedBlob? = nil @@ -468,59 +465,23 @@ actor FileCheckpointStore: CheckpointStore { lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && FileIdentity.matchesForMetadataReuse(lhs.identity, rhs.identity) && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && - lhs.children == rhs.children && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed + lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } } - var version: Int var id: UUID var rootPath: WorkspacePath var root: Node - // Older manifests and materialized snapshots have no trustworthy capture timestamp. + // Materialized snapshots have no trustworthy capture timestamp. var captureStartedAt: Date? = nil - var exactContentTimestamps: Bool? = nil - - private enum CodingKeys: String, CodingKey { - case version, id, rootPath, root, treeRoot, captureStartedAt, exactContentTimestamps - } - - init(version: Int, id: UUID, rootPath: WorkspacePath, root: Node, captureStartedAt: Date? = nil, exactContentTimestamps: Bool? = nil) { - self.version = version - self.id = id - self.rootPath = rootPath - self.root = root - self.captureStartedAt = captureStartedAt - self.exactContentTimestamps = exactContentTimestamps - } - - init(from decoder: any Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - version = try values.decode(Int.self, forKey: .version) - id = try values.decode(UUID.self, forKey: .id) - rootPath = try values.decode(WorkspacePath.self, forKey: .rootPath) - root = try values.decode(Node.self, forKey: version >= 3 ? .treeRoot : .root) - captureStartedAt = try values.decodeIfPresent(Date.self, forKey: .captureStartedAt) - exactContentTimestamps = try values.decodeIfPresent(Bool.self, forKey: .exactContentTimestamps) - } - - func encode(to encoder: any Encoder) throws { - var values = encoder.container(keyedBy: CodingKeys.self) - try values.encode(version, forKey: .version) - try values.encode(id, forKey: .id) - try values.encode(rootPath, forKey: .rootPath) - // Deliberately omit v1's required root key: even an already-admitted - // legacy operation must fail decoding before it can sweep live blobs. - try values.encode(root, forKey: version >= 3 ? .treeRoot : .root) - try values.encodeIfPresent(captureStartedAt, forKey: .captureStartedAt) - try values.encodeIfPresent(exactContentTimestamps, forKey: .exactContentTimestamps) - } + var exactContentTimestamps: Bool = false } /// Names are validated before entering the cache; no absolute paths are persisted. private struct TreeEntry: Codable, Sendable, Equatable { var name: String var kind: SnapshotManifest.Node.Kind - var permissions: POSIXPermissions? + var permissions: POSIXPermissions var contentHash: String? var contentSize: UInt64? var modificationDate: Date? @@ -532,6 +493,9 @@ actor FileCheckpointStore: CheckpointStore { init(_ node: SnapshotManifest.Node, name: String? = nil) throws { self.init(metadataOf: node, name: name ?? node.path.name) try Self.validate(self.name) + guard kind != .missing else { + throw WorkspaceError.storageCorrupted("missing entries cannot be stored in a directory tree") + } } /// Prior leaf metadata can also represent a file at the snapshot root. @@ -615,7 +579,6 @@ actor FileCheckpointStore: CheckpointStore { } else { parentManifest = nil } - try self.requireCurrentFormatUnlocked(workspaceId: draft.workspaceID) let batch = try self.makeBlobBatch(workspaceId: draft.workspaceID) defer { batch.close() } batch.summarizeInsertions = parentManifest == nil @@ -630,7 +593,7 @@ actor FileCheckpointStore: CheckpointStore { ) try self.finish(batch, workspaceId: draft.workspaceID) let manifest = SnapshotManifest( - version: 3, id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt, + id: snapshotID, rootPath: .root, root: root, captureStartedAt: captureStartedAt, exactContentTimestamps: await filesystem.supportsExactContentTimestamps ) let summary: ChangeSet.Summary @@ -658,13 +621,11 @@ actor FileCheckpointStore: CheckpointStore { } private func writeSnapshotUnlocked(_ snapshot: Snapshot, workspaceId: UUID, reusing previous: SnapshotManifest.Node?) throws { - try requireCurrentFormatUnlocked(workspaceId: workspaceId) let batch = try makeBlobBatch(workspaceId: workspaceId) defer { batch.close() } let root = try writeManifestNode(snapshot.entry, workspaceId: workspaceId, batch: batch, reusing: previous) try finish(batch, workspaceId: workspaceId) let manifest = SnapshotManifest( - version: 3, id: snapshot.id, rootPath: snapshot.rootPath, root: root @@ -676,21 +637,19 @@ actor FileCheckpointStore: CheckpointStore { let url = snapshotURL(id: id, workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return nil } let manifest = try decoder.decode(SnapshotManifest.self, from: Data(contentsOf: url)) - guard (1...3).contains(manifest.version), manifest.id == id else { + guard manifest.id == id else { throw WorkspaceError.storageCorrupted("snapshot manifest id does not match \(id)") } return manifest } - /// A frame keeps cached name-only entries intact. Only a requested old child - /// becomes a Node; legacy recursive manifests retain their embedded children. + /// A frame keeps cached name-only entries intact. Only a requested child becomes a Node. private struct CaptureDirectory { var path: WorkspacePath var name: String var childOffset: Int var previous: SnapshotManifest.Node? var previousEntries: [TreeEntry] - var legacyChildren: [SnapshotManifest.Node]? var children: [TreeEntry] = [] func previousEntry(named name: String) -> TreeEntry? { @@ -700,7 +659,7 @@ actor FileCheckpointStore: CheckpointStore { func previousNode(named name: String, at path: WorkspacePath) -> SnapshotManifest.Node? { guard let index = TreeEntry.index(of: name, in: previousEntries) else { return nil } - return legacyChildren?[index] ?? previousEntries[index].node(at: path) + return previousEntries[index].node(at: path) } } @@ -783,23 +742,11 @@ actor FileCheckpointStore: CheckpointStore { reusing previous: SnapshotManifest.Node?, at path: WorkspacePath, name: String, workspaceId: UUID ) throws -> CaptureDirectory { let entries: [TreeEntry] - let legacy: [SnapshotManifest.Node]? if let previous, previous.kind == .directory { - if let hash = previous.treeHash { - entries = try treeEntries(hash: hash, workspaceId: workspaceId, capturing: true) - legacy = nil - } else { - legacy = (previous.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) } - entries = try legacy!.map { try TreeEntry($0) } - for index in entries.indices.dropFirst() { - guard PathName(entries[index - 1].name) < PathName(entries[index].name) else { - throw WorkspaceError.storageCorrupted("snapshot directory \(path) contains duplicate child \(entries[index].name)") - } - } - } - } else { entries = []; legacy = nil } + entries = try treeEntries(hash: directoryHash(previous), workspaceId: workspaceId, capturing: true) + } else { entries = [] } return CaptureDirectory(path: path, name: name, childOffset: path.isRoot ? 1 : path.string.utf8.count + 1, - previous: previous, previousEntries: entries, legacyChildren: legacy) + previous: previous, previousEntries: entries) } private func finishCaptureDirectory( @@ -871,8 +818,7 @@ actor FileCheckpointStore: CheckpointStore { modificationDate.addingTimeInterval(2) < previousCaptureStartedAt, previous?.kind == .file, previous?.contentSize == info.size, - (previous?.identity == nil || info.identity == nil - || FileIdentity.matchesForMetadataReuse(previous?.identity, info.identity)), + FileIdentity.matchesForMetadataReuse(previous?.identity, info.identity), previous?.modificationDate != nil, previous?.modificationDate == info.modificationDate, let hash = previous?.contentHash, @@ -969,28 +915,31 @@ actor FileCheckpointStore: CheckpointStore { case .missing: return .missing(path: node.path) case .file: - guard let hash = node.contentHash, Self.isValidContentHash(hash) else { + guard let hash = node.contentHash, Self.isValidContentHash(hash), let size = node.contentSize else { throw WorkspaceError.storageCorrupted("snapshot file node \(node.path) has an invalid content hash") } return .file( path: node.path, - size: node.contentSize ?? 0, - permissions: node.permissions ?? .defaultFile, + size: size, + permissions: node.permissions, contentID: hash ) case .directory: return .directory( path: node.path, - permissions: node.permissions ?? .defaultDirectory, + permissions: node.permissions, children: depth == 0 ? [] : try children(of: node, workspaceId: workspaceId, capturing: capturing).map { try revisionEntry(from: $0, workspaceId: workspaceId, depth: depth.map { $0 - 1 }, capturing: capturing) } ) case .symlink: + guard let target = node.symlinkTarget else { + throw WorkspaceError.storageCorrupted("symlink \(node.path) has no target") + } return .symbolicLink( path: node.path, - target: node.symlinkTarget ?? "", - permissions: node.permissions ?? POSIXPermissions(0o777) + target: target, + permissions: node.permissions ) } } @@ -1001,17 +950,10 @@ actor FileCheckpointStore: CheckpointStore { let prefix = root.path.components var node = root for component in components.dropFirst(prefix.count) { - let name = PathName(component) guard node.kind == .directory else { return nil } - if let hash = node.treeHash { - let entries = try treeEntries(hash: hash, workspaceId: workspaceId) - guard let index = TreeEntry.index(of: component, in: entries) else { return nil } - node = entries[index].node(in: node.path) - } else { - // Legacy manifests already contain absolute child nodes. - guard let child = node.children?.first(where: { PathName($0.path.name) == name }) else { return nil } - node = child - } + let entries = try treeEntries(hash: directoryHash(node), workspaceId: workspaceId) + guard let index = TreeEntry.index(of: component, in: entries) else { return nil } + node = entries[index].node(in: node.path) } return node } @@ -1063,8 +1005,8 @@ actor FileCheckpointStore: CheckpointStore { return reused } return try persistDirectory(SnapshotManifest.Node( - kind: .directory, path: directory.path, permissions: directory.permissions, children: entries - ), workspaceId: workspaceId) + kind: .directory, path: directory.path, permissions: directory.permissions + ), children: entries, workspaceId: workspaceId) } } @@ -1084,23 +1026,26 @@ actor FileCheckpointStore: CheckpointStore { Snapshot.File( path: node.path, data: try materializedBlob(node, workspaceId: workspaceId, cache: cache), - permissions: node.permissions ?? POSIXPermissions(0o644), + permissions: node.permissions, contentID: hash ) ) case .symlink: + guard let target = node.symlinkTarget else { + throw WorkspaceError.storageCorrupted("symlink \(node.path) has no target") + } return .symlink( Snapshot.Symlink( path: node.path, - target: node.symlinkTarget ?? "", - permissions: node.permissions ?? POSIXPermissions(0o777) + target: target, + permissions: node.permissions ) ) case .directory: return .directory( Snapshot.Directory( path: node.path, - permissions: node.permissions ?? POSIXPermissions(0o755), + permissions: node.permissions, children: try children(of: node, workspaceId: workspaceId).map { try snapshotEntry(from: $0, workspaceId: workspaceId, cache: cache) } @@ -1293,7 +1238,6 @@ actor FileCheckpointStore: CheckpointStore { let root = try relocating(manifest.root, blobs: relocated, workspaceId: workspaceId) if root != manifest.root { manifest.root = root - manifest.version = 3 try write(manifest, to: snapshotURL(id: id, workspaceId: workspaceId), durability: .publication) } try collectArtifacts(from: manifest.root, workspaceId: workspaceId, blobs: &retainedBlobHashes, @@ -1525,8 +1469,7 @@ actor FileCheckpointStore: CheckpointStore { let old = try children(of: node, workspaceId: workspaceId) let new = try old.map { try relocating($0, blobs: blobs, workspaceId: workspaceId) } if old != new { - result.children = new - result = try persistDirectory(result, workspaceId: workspaceId, syncParent: true) + result = try persistDirectory(result, children: new, workspaceId: workspaceId, syncParent: true) } } return result @@ -1780,34 +1723,21 @@ actor FileCheckpointStore: CheckpointStore { func ensureWorkspaceDirectories(for workspaceId: UUID) async throws { try ensureDirectory(rootDirectory) try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { - let version = try ensureWorkspaceDirectoriesLocked(for: workspaceId) - if version == 1 { - try await withExclusiveLock(at: lifecycleLockURL(workspaceId: workspaceId)) { - try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { - try upgradeFormatUnlocked(workspaceId: workspaceId) - } - } - } + try ensureWorkspaceDirectoriesLocked(for: workspaceId) } } - private func ensureWorkspaceDirectoriesLocked(for workspaceId: UUID) throws -> Int { + private func ensureWorkspaceDirectoriesLocked(for workspaceId: UUID) throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) let existed = fileManager.fileExists(atPath: workspaceURL.path) - try ensureDirectory(workspaceURL) - let formatURL = workspaceURL.appendingPathComponent("format.json") - let version: Int - if fileManager.fileExists(atPath: formatURL.path) { - version = try validateWorkspaceFormatLocked(workspaceId) + if existed { + try validateWorkspaceFormatLocked(workspaceId) } else { - if existed, - !(try fileManager.contentsOfDirectory(at: workspaceURL, includingPropertiesForKeys: nil)).isEmpty - { - throw WorkspaceError.storageCorrupted("workspace store has no format version") - } - try Self.durableWrite(encoder.encode(StoreFormat(version: StoreFormat.currentVersion)), to: formatURL) - version = StoreFormat.currentVersion + try ensureDirectory(workspaceURL) + try Self.durableWrite(encoder.encode(StoreFormat()), + to: workspaceURL.appendingPathComponent("format.json")) } + // A prior initialization may have stopped after publishing the marker. let newObjects = !fileManager.fileExists(atPath: treesDirectoryURL(workspaceId: workspaceId).path) try ensureDirectory(checkpointsDirectoryURL(workspaceId: workspaceId)) try ensureDirectory(snapshotsDirectoryURL(workspaceId: workspaceId)) @@ -1819,28 +1749,24 @@ actor FileCheckpointStore: CheckpointStore { try Self.syncDirectory(rootDirectory) try Self.syncDirectory(rootDirectory.deletingLastPathComponent()) } - return version } func validateWorkspaceFormatIfPresent(_ workspaceId: UUID) async throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: workspaceURL.path) else { return } - try await withExclusiveLock(at: formatLockURL(workspaceId: workspaceId)) { - _ = try validateWorkspaceFormatLocked(workspaceId) + try await withSharedLock(at: formatLockURL(workspaceId: workspaceId)) { + try validateWorkspaceFormatLocked(workspaceId) } } - private func validateWorkspaceFormatLocked(_ workspaceId: UUID) throws -> Int { + private func validateWorkspaceFormatLocked(_ workspaceId: UUID) throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) let formatURL = workspaceURL.appendingPathComponent("format.json") - guard fileManager.fileExists(atPath: formatURL.path) else { - throw WorkspaceError.storageCorrupted("workspace store has no format version") + let data = fileManager.fileExists(atPath: formatURL.path) ? try Data(contentsOf: formatURL) : nil + guard let data, let format = try? decoder.decode(StoreFormat.self, from: data), + format == StoreFormat() else { + throw WorkspaceError.storageCorrupted("unsupported or missing format marker; delete the history directory at \(workspaceURL.path)") } - let format = try decoder.decode(StoreFormat.self, from: Data(contentsOf: formatURL)) - guard (1...StoreFormat.currentVersion).contains(format.version) else { - throw WorkspaceError.storageCorrupted("unsupported store format version \(format.version)") - } - return format.version } func workspaceDirectoryURL(workspaceId: UUID) -> URL { @@ -1887,17 +1813,12 @@ actor FileCheckpointStore: CheckpointStore { } } - /// Reads legacy bare-Int sidecars as epoch zero. Must hold the mutations lock. + /// Must hold the mutations lock. private func readMutationSequenceState(workspaceId: UUID) throws -> MutationSequenceState { let url = mutationSequenceURL(workspaceId: workspaceId) guard fileManager.fileExists(atPath: url.path) else { return MutationSequenceState() } let data = try Data(contentsOf: url) - let state: MutationSequenceState - if let legacy = try? decoder.decode(Int.self, from: data) { - state = MutationSequenceState(highWaterMark: legacy) - } else { - state = try decoder.decode(MutationSequenceState.self, from: data) - } + let state = try decoder.decode(MutationSequenceState.self, from: data) guard state.highWaterMark >= 0, state.retractionEpoch >= 0 else { throw WorkspaceError.storageCorrupted("negative mutation sequence or retraction epoch") } @@ -1934,6 +1855,7 @@ actor FileCheckpointStore: CheckpointStore { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("lifecycle.lock", isDirectory: false) } + /// Serializes first-time directory and marker publication across processes. private func formatLockURL(workspaceId: UUID) -> URL { rootDirectory.appendingPathComponent(".\(workspaceId.uuidString).format.lock", isDirectory: false) } @@ -1958,11 +1880,9 @@ actor FileCheckpointStore: CheckpointStore { var message: String } - private struct StoreFormat: Codable { - // Existing v1 manifests and sequence sidecars remain readable. The v2 writer - // fence is published before new directory objects or packs (see README). - static let currentVersion = 2 - var version: Int + private struct StoreFormat: Codable, Equatable { + var format = "workspace-history" + var version = 1 } // One combined estimated-decoded budget per store, split between nodes and write catalog. @@ -2136,10 +2056,17 @@ actor FileCheckpointStore: CheckpointStore { func children(of node: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) throws -> [SnapshotManifest.Node] { guard node.kind == .directory else { return [] } - guard let hash = node.treeHash else { return (node.children ?? []).sorted { PathName($0.path.name) < PathName($1.path.name) } } + let hash = try directoryHash(node) return try treeEntries(hash: hash, workspaceId: workspaceId, capturing: capturing).map { $0.node(in: node.path) } } + private func directoryHash(_ node: SnapshotManifest.Node) throws -> String { + guard let hash = node.treeHash else { + throw WorkspaceError.storageCorrupted("directory \(node.path) has no tree hash") + } + return hash + } + private func treeEntries(hash: String, workspaceId: UUID, capturing: Bool = false) throws -> [TreeEntry] { guard Self.isValidContentHash(hash) else { throw WorkspaceError.storageCorrupted("invalid tree hash") } let url = treesDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash) @@ -2154,6 +2081,9 @@ actor FileCheckpointStore: CheckpointStore { var previous: PathName? for entry in entries { try TreeEntry.validate(entry.name) + guard entry.kind != .missing else { + throw WorkspaceError.storageCorrupted("missing entries cannot be stored in a directory tree") + } guard previous.map({ $0 < PathName(entry.name) }) ?? true else { throw WorkspaceError.storageCorrupted("unsorted or duplicate tree child") } @@ -2165,9 +2095,9 @@ actor FileCheckpointStore: CheckpointStore { } func persistDirectory( - _ node: SnapshotManifest.Node, workspaceId: UUID, syncParent: Bool = false + _ node: SnapshotManifest.Node, children: [SnapshotManifest.Node], workspaceId: UUID, syncParent: Bool = false ) throws -> SnapshotManifest.Node { - let entries = try (node.children ?? []).sorted { + let entries = try children.sorted { PathName($0.path.name) < PathName($1.path.name) }.map { try TreeEntry($0) } return try persistDirectory(node, entries: entries, workspaceId: workspaceId, syncParent: syncParent) @@ -2185,7 +2115,6 @@ actor FileCheckpointStore: CheckpointStore { } cacheTree(entries, hash: hash, workspaceId: workspaceId, capturing: capturing) var reference = node - reference.children = nil reference.treeHash = hash return reference } @@ -2202,7 +2131,10 @@ actor FileCheckpointStore: CheckpointStore { return (packsDirectoryURL(workspaceId: workspaceId).appendingPathComponent(packed.pack + ".pack"), packed.offset, packed.size) } let url = blobURL(hash: hash, workspaceId: workspaceId) - return (url, 0, try node.contentSize ?? fileSize(at: url)) + guard let size = node.contentSize else { + throw WorkspaceError.storageCorrupted("file \(node.path) has no content size") + } + return (url, 0, size) } private func blobExists(_ node: SnapshotManifest.Node, workspaceId: UUID, batch: BlobBatch) -> Bool { @@ -2295,7 +2227,7 @@ actor FileCheckpointStore: CheckpointStore { guard let node = try manifestNode(at: current, in: manifest.root, workspaceId: workspaceId) else { return nil } if node.kind == .file { return .file(path: current, size: try blobLocation(node, workspaceId: workspaceId).size, - permissions: node.permissions ?? .defaultFile, contentID: node.contentHash ?? "") + permissions: node.permissions, contentID: node.contentHash ?? "") } return try revisionEntry(from: node, workspaceId: workspaceId, depth: 0) } @@ -2335,7 +2267,7 @@ actor FileCheckpointStore: CheckpointStore { func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) async throws -> ChangeSet { if old == new { return ChangeSet(changes: []) } if old.kind == .directory, new.kind == .directory { - var changes: [FileChange] = (old.permissions ?? .defaultDirectory) == (new.permissions ?? .defaultDirectory) ? [] : [.init(path: new.path, kind: .directory, effect: .modified)] + var changes: [FileChange] = (old.permissions) == (new.permissions) ? [] : [.init(path: new.path, kind: .directory, effect: .modified)] var omissions: [TextDiffOmission] = [] if old.treeHash != nil, old.treeHash == new.treeHash { return ChangeSet(changes: changes) } let before = Dictionary(uniqueKeysWithValues: try children(of: old, workspaceId: workspaceId, capturing: capturing).map { (PathName($0.path.name), $0) }) @@ -2366,26 +2298,6 @@ actor FileCheckpointStore: CheckpointStore { return try readBlob(node, workspaceId: workspaceId) } - func requireCurrentFormatUnlocked(workspaceId: UUID) throws { - let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("format.json") - guard try read(StoreFormat.self, from: url).version == StoreFormat.currentVersion else { - throw WorkspaceError.storageCorrupted("store format changed after preparation") - } - } - - /// v1 manifests remain readable. Publish the writer fence before any v2 objects. - /// Migration runs under format -> lifecycle -> mutation locks, in that order. - private func upgradeFormatUnlocked(workspaceId: UUID) throws { - let url = workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("format.json") - let format = try read(StoreFormat.self, from: url) - guard (1...StoreFormat.currentVersion).contains(format.version) else { - throw WorkspaceError.storageCorrupted("unsupported store format version \(format.version)") - } - if format.version < StoreFormat.currentVersion { - try Self.durableWrite(encoder.encode(StoreFormat(version: StoreFormat.currentVersion)), to: url) - } - } - private nonisolated static func durableWrite( _ data: Data, to url: URL, syncParent: Bool = true, publication: Bool = false ) throws { diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index 6eef2f1..f1ed8a6 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -218,9 +218,6 @@ public protocol FileSystem: AnyObject, Sendable { /// A filesystem that can push invalidations when its contents change outside Workspace APIs. public protocol FileSystemChangeSource: FileSystem { - /// Emits an unscoped invalidation. Workspace consumes `changeSignals()`, whose - /// default implementation adapts this stream to `.rescan`. - func changes() async throws -> AsyncStream /// Emits scoped invalidations using the `FileSystemChangeSignal` contract. func changeSignals() async throws -> AsyncStream } @@ -268,20 +265,6 @@ public enum FileSystemChangeSignal: Sendable, Equatable { } } -extension FileSystemChangeSource { - /// Compatibility for sources that only provide unscoped invalidations. - public func changeSignals() async throws -> AsyncStream { - let stream = try await changes() - return AsyncStream { continuation in - let task = Task { - for await _ in stream where !Task.isCancelled { continuation.yield(.rescan) } - continuation.finish() - } - continuation.onTermination = { _ in task.cancel() } - } - } -} - /// Resolves links in the backend's namespace without requiring the target to exist. /// Missing parent suffixes are retained for quota preflight without creating them. package protocol PotentialPathResolvingFileSystem: FileSystem { @@ -373,8 +356,6 @@ extension FileSystem { } } -typealias FileSystemCapabilities = FileSystemFeatures - /// Backend differences relevant to copy-on-write edit previews. package struct FileSystemPreviewSemantics: Sendable { package var preservesOverwritePermissions: Bool diff --git a/Sources/Workspace/FS/FileSystemTraversal.swift b/Sources/Workspace/FS/FileSystemTraversal.swift index 409dd85..d2186bf 100644 --- a/Sources/Workspace/FS/FileSystemTraversal.swift +++ b/Sources/Workspace/FS/FileSystemTraversal.swift @@ -16,8 +16,8 @@ import Glibc /// when the replacement link points inside the root. Very deep local subtrees fall /// back to ordinary per-path operations with a root identity check at the handoff. /// -/// This optional package capability keeps custom backends source-compatible and lets -/// authorization decorators retain per-entry checks through the per-path fallback. +/// This optional package capability accelerates traversal while authorization +/// decorators retain per-entry checks through per-path operations. package protocol TraversingFileSystem: FileSystem { /// Forwarders return false when their base needs the per-path fallback. var supportsScopedWalk: Bool { get } diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index fb9d3d3..e21dc7e 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -732,7 +732,7 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti } // Prefer backend-owned resolution so physical parents and absolute targets retain their -// backend meaning, including through decorators. Older backends use the heuristic below. +// backend meaning, including through decorators. Backends without this capability use the heuristic below. struct FileSystemPathResolver: Sendable { private let base: any FileSystem @@ -811,10 +811,6 @@ struct FileSystemPathResolver: Sendable { extension LimitedFileSystem: CompleteFileSystemChangeSource { package var changeSignalsAreComplete: Bool { get async { await base.changeSignalsAreComplete } } - public func changes() async throws -> AsyncStream { - unscopedChanges(try await changeSignals()) - } - public func changeSignals() async throws -> AsyncStream { guard let source = base as? any FileSystemChangeSource else { throw WorkspaceError.unsupported("filesystem change notifications are not supported") diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 78cf7e0..95c11ff 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -1095,12 +1095,6 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem #if os(macOS) extension LocalFileSystem: FileSystemChangeSource { - /// See ``FileSystemChangeSource/changes()``. - public func changes() async throws -> AsyncStream { - let root = try requireConfiguration().rootURL - return try FSEventsChangeStream.observe(root) - } - public func changeSignals() async throws -> AsyncStream { let configuration = try requireConfiguration() return try FSEventsChangeStream.observeSignals(configuration.rootURL, resolvedRootPath: configuration.resolvedRootPath) @@ -1273,11 +1267,6 @@ enum FSEventsChangeStream { } #elseif os(Linux) extension LocalFileSystem: FileSystemChangeSource { - /// See ``FileSystemChangeSource/changes()``. - public func changes() async throws -> AsyncStream { - try DirectoryChangeStream.observe(requireConfiguration().rootURL, recursive: true) - } - public func changeSignals() async throws -> AsyncStream { try DirectoryChangeStream.observeSignals(requireConfiguration().rootURL, recursive: true) } diff --git a/Sources/Workspace/FS/MountedFileSystem.swift b/Sources/Workspace/FS/MountedFileSystem.swift index 4d4a672..3ef4a42 100644 --- a/Sources/Workspace/FS/MountedFileSystem.swift +++ b/Sources/Workspace/FS/MountedFileSystem.swift @@ -513,10 +513,6 @@ extension MountedFileSystem: CompleteFileSystemChangeSource { return true } } - public func changes() async throws -> AsyncStream { - unscopedChanges(try await changeSignals()) - } - public func changeSignals() async throws -> AsyncStream { let id = UUID() let (topology, updates) = AsyncStream<[Mount]>.makeStream(bufferingPolicy: .bufferingNewest(1)) diff --git a/Sources/Workspace/FS/SandboxFileSystem.swift b/Sources/Workspace/FS/SandboxFileSystem.swift index 5ac1eb1..f5be5a6 100644 --- a/Sources/Workspace/FS/SandboxFileSystem.swift +++ b/Sources/Workspace/FS/SandboxFileSystem.swift @@ -171,15 +171,8 @@ public final class SandboxFileSystem: FileSystem, TraversingFileSystem, PreviewS } } -typealias SandboxFilesystem = SandboxFileSystem - - extension SandboxFileSystem: CompleteFileSystemChangeSource { package var changeSignalsAreComplete: Bool { get async { await (backing as any FileSystem).changeSignalsAreComplete } } - public func changes() async throws -> AsyncStream { - unscopedChanges(try await changeSignals()) - } - public func changeSignals() async throws -> AsyncStream { guard let source = (backing as any FileSystem) as? any FileSystemChangeSource else { throw WorkspaceError.unsupported("filesystem change notifications are not supported") diff --git a/Sources/Workspace/FS/SecurityScopedFileSystem.swift b/Sources/Workspace/FS/SecurityScopedFileSystem.swift index b51ad15..bcc7cdf 100644 --- a/Sources/Workspace/FS/SecurityScopedFileSystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFileSystem.swift @@ -297,14 +297,8 @@ public final class SecurityScopedFileSystem: FileSystem, TraversingFileSystem, P #endif } -typealias SecurityScopedFilesystem = SecurityScopedFileSystem - extension SecurityScopedFileSystem: CompleteFileSystemChangeSource { package var changeSignalsAreComplete: Bool { get async { await (backing as any FileSystem).changeSignalsAreComplete } } - public func changes() async throws -> AsyncStream { - unscopedChanges(try await changeSignals()) - } - public func changeSignals() async throws -> AsyncStream { guard let source = (backing as any FileSystem) as? any FileSystemChangeSource else { throw WorkspaceError.unsupported("filesystem change notifications are not supported") diff --git a/Sources/Workspace/Support/FileChangeStreams.swift b/Sources/Workspace/Support/FileChangeStreams.swift index 9b3eeea..54bc87e 100644 --- a/Sources/Workspace/Support/FileChangeStreams.swift +++ b/Sources/Workspace/Support/FileChangeStreams.swift @@ -270,7 +270,7 @@ private final class LinuxDirectoryObservation: @unchecked Sendable { } #endif -/// Retains compatibility for checkpoint stores and callers of the original Void stream. +/// Coalesces checkpoint-store notifications, whose consumers reload checkpoint metadata. func unscopedChanges(_ stream: AsyncStream) -> AsyncStream { AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in let task = Task { diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 751c70f..8d2ab87 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -536,10 +536,6 @@ extension AuthorizedFileSystem: CompleteFileSystemChangeSource { /// Signals for paths the authorizer hides are dropped, and decisions can change over time, so /// consumers must not treat this stream as complete coverage of the base. package var changeSignalsAreComplete: Bool { get async { false } } - public func changes() async throws -> AsyncStream { - unscopedChanges(try await changeSignals()) - } - public func changeSignals() async throws -> AsyncStream { guard let source = base as? any FileSystemChangeSource else { throw WorkspaceError.unsupported("filesystem change notifications are not supported") diff --git a/Sources/Workspace/TransactionTree.swift b/Sources/Workspace/TransactionTree.swift index 1e2a183..a343d2d 100644 --- a/Sources/Workspace/TransactionTree.swift +++ b/Sources/Workspace/TransactionTree.swift @@ -253,7 +253,7 @@ actor TransactionRevisionContent { } func read(_ path: WorkspacePath) async throws -> Data { - guard let data = try await store.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId, + guard let data = try await store.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceID, path: path, offset: 0, length: nil) else { throw WorkspaceError.revisionDataNotFound(checkpoint.snapshotId) } @@ -279,7 +279,7 @@ final class TransactionRevisionPins: @unchecked Sendable { private var pins: [UUID: (workspace: UUID, checkpoint: UUID)] = [:] func insert(_ checkpoint: Checkpoint) -> UUID { let token = UUID() - lock.withLock { pins[token] = (checkpoint.workspaceId, checkpoint.id) } + lock.withLock { pins[token] = (checkpoint.workspaceID, checkpoint.id) } return token } func remove(_ token: UUID) { _ = lock.withLock { pins.removeValue(forKey: token) } } diff --git a/Sources/Workspace/Workspace+Checkpoints.swift b/Sources/Workspace/Workspace+Checkpoints.swift index bef33c9..792831d 100644 --- a/Sources/Workspace/Workspace+Checkpoints.swift +++ b/Sources/Workspace/Workspace+Checkpoints.swift @@ -77,7 +77,7 @@ extension Workspace { func checkpointComparison(_ checkpoint: Checkpoint) async throws -> (RevisionComparison, TransactionRevisionPin) { let store = store let (revision, pin) = try await TransactionRevisionGate.shared.withLock { - guard let revision = try await store.loadCapturedRevision(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) else { + guard let revision = try await store.loadCapturedRevision(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceID) else { throw WorkspaceError.revisionDataNotFound(checkpoint.snapshotId) } return (revision, TransactionRevisionPin(checkpoint)) diff --git a/Sources/Workspace/Workspace+TransactionStorage.swift b/Sources/Workspace/Workspace+TransactionStorage.swift index c054b93..6ffd66d 100644 --- a/Sources/Workspace/Workspace+TransactionStorage.swift +++ b/Sources/Workspace/Workspace+TransactionStorage.swift @@ -33,7 +33,7 @@ extension Workspace { func storedTransactionTree(_ checkpoint: Checkpoint, in store: any CheckpointStore) async throws -> TransactionTree { if usesMemoryPersistence, - let snapshot = try await store.loadSnapshot(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) { + let snapshot = try await store.loadSnapshot(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceID) { var nodes: [WorkspacePath: MergeNode] = [:] Self.flattenMergeNodes(snapshot.entry, into: &nodes) var tree = TransactionTree(nodes: nodes, load: { _ in throw WorkspaceError.readOnly }) @@ -42,7 +42,7 @@ extension Workspace { } return try await TransactionRevisionGate.shared.withLock { let content = TransactionRevisionContent(checkpoint: checkpoint, store: store) - guard let index = try await store.loadRevisionIndex(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) else { + guard let index = try await store.loadRevisionIndex(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceID) else { throw WorkspaceError.revisionDataNotFound(checkpoint.snapshotId) } var tree = TransactionTree(index: index, load: { try await content.read($0) }) diff --git a/Sources/Workspace/WorkspaceEvent.swift b/Sources/Workspace/WorkspaceEvent.swift index d5d533e..b10d653 100644 --- a/Sources/Workspace/WorkspaceEvent.swift +++ b/Sources/Workspace/WorkspaceEvent.swift @@ -66,14 +66,6 @@ extension Workspace { } } - /// Compatibility spelling for observing future workspace changes at a path. - public func watchChanges( - at path: WorkspacePath = .root, - recursive: Bool = true - ) async -> AsyncStream { - await events(.init(path: path, recursive: recursive, includeCheckpoints: false)) - } - private static func matches(_ event: WorkspaceEvent, filter: WorkspaceEvent.Filter) -> Bool { switch event { case .checkpoint: diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index 8a55c2c..4021c4d 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -65,7 +65,7 @@ extension Workspace { try await ensureLoaded() try await reconcileCheckpointsWithStore() let checkpoint = try checkpointOrThrow(id: id) - return try await loadSnapshotOrThrow(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId) + return try await loadSnapshotOrThrow(id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceID) } } @@ -103,7 +103,7 @@ extension Workspace { case let .checkpoint(id): let access = try await checkpointAccess(id) guard let index = try await store.loadRevisionIndex( - id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceId + id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceID ) else { throw WorkspaceError.revisionDataNotFound(access.checkpoint.snapshotId) } return ResolvedRevision(index: index, readRange: access.readRange) } @@ -140,7 +140,7 @@ extension Workspace { case let .checkpoint(id): let access = try await checkpointAccess(id) let file = try await store.resolveSnapshotFile( - id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceId, path: path + id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceID, path: path ) return ResolvedReadableFile(size: file.size, readRange: { offset, length in try await access.readRange(file.path, offset, length) @@ -160,7 +160,7 @@ extension Workspace { let store = store return CheckpointAccess(checkpoint: checkpoint, readRange: { path, offset, length in guard let data = try await store.readSnapshotFile( - id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId, + id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceID, path: path, offset: offset, length: length ) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOENT)) } return data @@ -170,7 +170,7 @@ extension Workspace { private func checkpointEntry(_ id: UUID, at path: WorkspacePath, includeChildren: Bool) async throws -> RevisionIndex.Entry? { let access = try await checkpointAccess(id) return try await store.snapshotEntry( - id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceId, + id: access.checkpoint.snapshotId, workspaceId: access.checkpoint.workspaceID, path: path, includeChildren: includeChildren ) } diff --git a/Sources/Workspace/WorkspaceSearch.swift b/Sources/Workspace/WorkspaceSearch.swift index 882375d..aaac799 100644 --- a/Sources/Workspace/WorkspaceSearch.swift +++ b/Sources/Workspace/WorkspaceSearch.swift @@ -22,15 +22,6 @@ public struct SearchRequest: Sendable, Codable, Equatable { self.maxMatches = maxMatches self.maxOutputBytes = maxOutputBytes } - - private enum CodingKeys: String, CodingKey { case maxFileBytes, maxFiles, maxMatches, maxOutputBytes } - public init(from decoder: Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - maxFileBytes = try values.decode(UInt64.self, forKey: .maxFileBytes) - maxFiles = try values.decode(Int.self, forKey: .maxFiles) - maxMatches = try values.decode(Int.self, forKey: .maxMatches) - maxOutputBytes = try values.decodeIfPresent(Int.self, forKey: .maxOutputBytes) - } } /// Maximum context on each side is 100 lines; negative requests are rejected. @@ -56,16 +47,6 @@ public struct SearchRequest: Sendable, Codable, Equatable { self.limits = limits self.matchOffset = matchOffset } - - private enum CodingKeys: String, CodingKey { case files, pattern, contextLines, limits, matchOffset } - public init(from decoder: Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - files = try values.decode(FileSelection.self, forKey: .files) - pattern = try values.decode(TextPattern.self, forKey: .pattern) - contextLines = try values.decode(Int.self, forKey: .contextLines) - limits = try values.decode(Limits.self, forKey: .limits) - matchOffset = try values.decodeIfPresent(Int.self, forKey: .matchOffset) ?? 0 - } } public struct SearchResult: Sendable, Codable, Equatable { @@ -110,20 +91,6 @@ public struct SearchResult: Sendable, Codable, Equatable { saturatedByteSum($0, UInt64($1.text.utf8.count)) } } - - private enum CodingKeys: String, CodingKey { case path, lineNumber, line, before, after, truncated, totalBytes } - public init(from decoder: Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - self.init( - path: try values.decode(WorkspacePath.self, forKey: .path), - lineNumber: try values.decode(Int.self, forKey: .lineNumber), - line: try values.decode(String.self, forKey: .line), - before: try values.decode([ContextLine].self, forKey: .before), - after: try values.decode([ContextLine].self, forKey: .after), - truncated: try values.decodeIfPresent(Bool.self, forKey: .truncated) ?? false, - totalBytes: try values.decodeIfPresent(UInt64.self, forKey: .totalBytes) - ) - } } public struct SkippedFile: Sendable, Codable, Equatable { @@ -176,20 +143,9 @@ public struct SearchResult: Sendable, Codable, Equatable { } private enum CodingKeys: String, CodingKey { - case matches, skippedFiles, searchedFileCount, truncation, truncated, totalMatches, nextMatchOffset - } - public init(from decoder: Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - let matches = try values.decode([Match].self, forKey: .matches) - self.init( - matches: matches, - skippedFiles: try values.decode([SkippedFile].self, forKey: .skippedFiles), - searchedFileCount: try values.decode(Int.self, forKey: .searchedFileCount), - truncation: try values.decodeIfPresent(Truncation.self, forKey: .truncation), - totalMatches: try values.decodeIfPresent(Int.self, forKey: .totalMatches) ?? matches.count, - nextMatchOffset: try values.decodeIfPresent(Int.self, forKey: .nextMatchOffset) - ) + case matches, skippedFiles, searchedFileCount, truncation, totalMatches, nextMatchOffset } + private enum ComputedCodingKeys: String, CodingKey { case truncated } public func encode(to encoder: Encoder) throws { var values = encoder.container(keyedBy: CodingKeys.self) @@ -197,7 +153,8 @@ public struct SearchResult: Sendable, Codable, Equatable { try values.encode(skippedFiles, forKey: .skippedFiles) try values.encode(searchedFileCount, forKey: .searchedFileCount) try values.encodeIfPresent(truncation, forKey: .truncation) - try values.encode(truncated, forKey: .truncated) + var computed = encoder.container(keyedBy: ComputedCodingKeys.self) + try computed.encode(truncated, forKey: .truncated) try values.encode(totalMatches, forKey: .totalMatches) try values.encodeIfPresent(nextMatchOffset, forKey: .nextMatchOffset) } diff --git a/Sources/Workspace/WorkspaceToolBudget.swift b/Sources/Workspace/WorkspaceToolBudget.swift index fc063d2..75e1177 100644 --- a/Sources/Workspace/WorkspaceToolBudget.swift +++ b/Sources/Workspace/WorkspaceToolBudget.swift @@ -1,7 +1,7 @@ import Foundation -/// Adapter envelopes retain the original ChangeSet/EditResult keys. Statistics -/// cover the whole selected operation; the nested ChangeSet describes this page. +/// Adapter statistics cover the whole selected operation; the nested ChangeSet +/// describes this page. struct ToolChangesOutput: Encodable { var changes = ChangeSet() var statistics: ChangeSet.Statistics diff --git a/Tests/WorkspaceTests/ChangeScalingTests.swift b/Tests/WorkspaceTests/ChangeScalingTests.swift index 3aa9764..35d1a71 100644 --- a/Tests/WorkspaceTests/ChangeScalingTests.swift +++ b/Tests/WorkspaceTests/ChangeScalingTests.swift @@ -250,7 +250,6 @@ final class SignallingFileSystem: FileSystemChangeSource, ExactTimestampFileSyst if let source { return try await source.changeSignals() } return stream } - func changes() async throws -> AsyncStream { unscopedChanges(stream) } func stat(path: WorkspacePath) async throws -> FileInfo { try await base.stat(path: path) } func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } func readFile(path: WorkspacePath) async throws -> Data { try await base.readFile(path: path) } diff --git a/Tests/WorkspaceTests/ChangeSoundnessTests.swift b/Tests/WorkspaceTests/ChangeSoundnessTests.swift index e8b16c8..8922af6 100644 --- a/Tests/WorkspaceTests/ChangeSoundnessTests.swift +++ b/Tests/WorkspaceTests/ChangeSoundnessTests.swift @@ -18,7 +18,6 @@ final class CoarseFileSystem: FileSystem, FileSystemChangeSource, Sendable { } func signal(_ s: FileSystemChangeSignal) { continuation.yield(s) } func changeSignals() async throws -> AsyncStream { stream } - func changes() async throws -> AsyncStream { unscopedChanges(stream) } func coarse(_ info: FileInfo) -> FileInfo { var i = info i.identity = mode == .coarse ? nil : FileIdentity(device: 1, inode: info.path == "/a" ? 1 : 2) @@ -315,7 +314,6 @@ final class FailingSignalFS: FileSystem, FileSystemChangeSource, @unchecked Send func failNext() { lock.withLock { fail = true } } func signal(_ s: FileSystemChangeSignal) { continuation.yield(s) } func changeSignals() async throws -> AsyncStream { stream } - func changes() async throws -> AsyncStream { unscopedChanges(stream) } func stat(path: WorkspacePath) async throws -> FileInfo { if lock.withLock({ let f = fail; fail = false; return f }) { throw TestFileSystemError.injected } return try await base.stat(path: path) diff --git a/Tests/WorkspaceTests/CheckpointFormatTests.swift b/Tests/WorkspaceTests/CheckpointFormatTests.swift index 01e187a..0b75a8d 100644 --- a/Tests/WorkspaceTests/CheckpointFormatTests.swift +++ b/Tests/WorkspaceTests/CheckpointFormatTests.swift @@ -4,68 +4,6 @@ import Testing @Suite("Checkpoint tree format") struct CheckpointFormatTests { - // This fixture was emitted by the original v1 FileCheckpointStore, including - // its mutation log and sequence sidecar. It is deliberately not re-encoded here. - @Test func `v1 fixture remains readable compactable and appendable with a v2 writer fence`() async throws { - let directory = try TestSupport.temporaryDirectory("LegacyStore") - defer { TestSupport.remove(directory) } - let source = try #require(Bundle.module.url(forResource: "v1", withExtension: nil, subdirectory: "Fixtures")) - let root = directory.appendingPathComponent("store") - try FileManager.default.copyItem(at: source, to: root) - let id = UUID(uuidString: "10000000-0000-0000-0000-000000000001")! - let store = FileCheckpointStore(rootDirectory: root) - let old = try await store.listCheckpoints(workspaceId: id) - #expect(old.count == 2) - let legacyFile = try await store.resolveSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: "/link") - #expect(legacyFile.path == "/nested/deep/file") - #expect(try await store.readSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: legacyFile.path, offset: 0, length: nil) == Data("nested legacy\n".utf8)) - - #expect(try await store.readSnapshotFile(id: old[0].snapshotId, workspaceId: id, path: "/note", offset: 0, length: nil) == Data("legacy checkpoint\n".utf8)) - #expect(try await store.readSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: "/note", offset: 0, length: nil) == Data("legacy updated\n".utf8)) - let dry = try await store.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: true) - let compacted = try await store.compact(workspaceId: id, retaining: .latest(1, preservingLabeled: false), dryRun: false) - #expect(dry.report.after == compacted.report.after) - #expect(compacted.checkpoints.first?.parentID == nil) - // Legacy nil directory permissions mean defaultDirectory, not a modification. - let legacyURL = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(old[1].snapshotId).json") - var legacyJSON = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: legacyURL)) as? [String: Any]) - var legacyRoot = try #require(legacyJSON["root"] as? [String: Any]) - legacyRoot.removeValue(forKey: "permissions") - legacyJSON["root"] = legacyRoot - try JSONSerialization.data(withJSONObject: legacyJSON).write(to: legacyURL) - let restored = InMemoryFileSystem() - let legacySnapshot = try #require(await store.loadSnapshot(id: old[1].snapshotId, workspaceId: id)) - try await Snapshot.restore(legacySnapshot, to: restored) - let unchanged = try await store.captureRevision(from: restored, draft: draft(id, parent: old[1].id)) - #expect(unchanged.summary.changedPathCount == 0) - - let next = try await store.saveRevision(snapshot(count: 4), draft: draft(id, parent: old[1].id)) - #expect(next.mutationCursor == 2) - let mutation = try await store.appendMutation(Mutation(sequence: 0, workspaceID: id, operation: .edit, changes: ChangeSet())) - #expect(mutation.sequence == 3) - let reopened = FileCheckpointStore(rootDirectory: root) - #expect(try await reopened.readSnapshotFile(id: old[1].snapshotId, workspaceId: id, path: "/note", offset: 0, length: nil) == Data("legacy updated\n".utf8)) - #expect(try await reopened.loadSnapshot(id: next.snapshotId, workspaceId: id)?.entry == snapshot(count: 4).entry) - let format = root.appendingPathComponent(id.uuidString).appendingPathComponent("format.json") - // The v1 writer's actual admission rule: refuse any version other than 1, - // before creating or changing checkpoint/mutation artifacts. - struct V1Format: Decodable { var version: Int } - func v1WriterAdmission() throws { - let value = try JSONDecoder().decode(V1Format.self, from: Data(contentsOf: format)) - guard value.version == 1 else { throw WorkspaceError.storageCorrupted("unsupported store format version \(value.version)") } - } - #expect(throws: WorkspaceError.self) { try v1WriterAdmission() } - let newManifest = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(next.snapshotId).json") - #expect(throws: DecodingError.self) { - _ = try JSONDecoder().decode(V1SnapshotManifest.self, from: Data(contentsOf: newManifest)) - } - // The copied legacy decoder also succeeds on the real legacy fixture. - let oldManifest = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(old[1].snapshotId).json") - #expect(try JSONDecoder().decode(V1SnapshotManifest.self, from: Data(contentsOf: oldManifest)).version == 2) - - #expect(try await reopened.listMutations(workspaceId: id).map(\.sequence) == [1, 2, 3]) - } - @Test func `one changed leaf writes only its ancestor trees and reopens correctly`() async throws { let root = try TestSupport.temporaryDirectory("TreeReuse") let files = try TestSupport.temporaryDirectory("TreeReuseFiles") @@ -98,7 +36,7 @@ struct CheckpointFormatTests { // is still reported by operations that need the complete snapshot. let manifestURL = root.appendingPathComponent(id.uuidString).appendingPathComponent("snapshots/\(third.snapshotId.uuidString).json") let manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: manifestURL)) as? [String: Any]) - let rootNode = try #require(manifest["treeRoot"] as? [String: Any]) + let rootNode = try #require(manifest["root"] as? [String: Any]) let rootHash = try #require(rootNode["treeHash"] as? String) let children = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: trees.appendingPathComponent(rootHash))) as? [[String: Any]]) let unrelated = try #require(children.first { ($0["name"] as? String) == "dir11" }) @@ -198,14 +136,14 @@ struct CheckpointFormatTests { try Data(oldIndex.replacingOccurrences(of: oldID, with: newID).utf8).write(to: packs.appendingPathComponent(newID + ".json")) let manifestURL = directory.appendingPathComponent("snapshots/\(second.snapshotId.uuidString).json") var manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: manifestURL)) as? [String: Any]) - var node = try #require(manifest["treeRoot"] as? [String: Any]) + var node = try #require(manifest["root"] as? [String: Any]) let treeID = try #require(node["treeHash"] as? String) let oldTree = try String(contentsOf: directory.appendingPathComponent("trees/" + treeID), encoding: .utf8) let newTree = Data(oldTree.replacingOccurrences(of: oldID, with: newID).utf8) let newTreeID = SHA256.hexDigest(of: newTree) try newTree.write(to: directory.appendingPathComponent("trees/" + newTreeID)) node["treeHash"] = newTreeID - manifest["treeRoot"] = node + manifest["root"] = node try JSONSerialization.data(withJSONObject: manifest).write(to: manifestURL) let reopened = FileCheckpointStore(rootDirectory: root) let dry = try await reopened.compact(workspaceId: id, retaining: .all, dryRun: true) @@ -253,7 +191,7 @@ struct CheckpointFormatTests { #expect(try await reader.loadSnapshot(id: recovered.snapshotId, workspaceId: id)?.entry == snapshot(count: 100).entry) } - @Test func `name trees deduplicate relocated subtrees and reject invalid names`() async throws { + @Test func `name trees deduplicate relocated subtrees and reject invalid entries`() async throws { let root = try TestSupport.temporaryDirectory("RelativeTrees") defer { TestSupport.remove(root) } let id = UUID() @@ -288,16 +226,20 @@ struct CheckpointFormatTests { let path = WorkspacePath(normalizing: "/\(name)/deep/file") #expect(try await store.readSnapshotFile(id: checkpoint.snapshotId, workspaceId: id, path: path, offset: 0, length: nil) == Data("same".utf8)) } - // Independent malformed tree objects: each hash is valid, only the name is invalid. + // Each object has a valid hash but an invalid name or child kind. let manifestURL = directory.appendingPathComponent("snapshots/\(second.snapshotId).json") var manifest = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: manifestURL)) as? [String: Any]) - var node = try #require(manifest["treeRoot"] as? [String: Any]) - for name in ["", ".", "..", "a/b", "a/\u{301}b", "a\0b"] { - let bytes = try JSONSerialization.data(withJSONObject: [["name": name, "kind": "directory"]]) + var node = try #require(manifest["root"] as? [String: Any]) + let invalidEntries = ["", ".", "..", "a/b", "a/\u{301}b", "a\0b"].map { ($0, "directory") } + + [("absent", "missing")] + for (name, kind) in invalidEntries { + let bytes = try JSONSerialization.data(withJSONObject: [[ + "name": name, "kind": kind, "permissions": ["rawValue": 493] + ]]) let hash = SHA256.hexDigest(of: bytes) try bytes.write(to: trees.appendingPathComponent(hash)) node["treeHash"] = hash - manifest["treeRoot"] = node + manifest["root"] = node try JSONSerialization.data(withJSONObject: manifest).write(to: manifestURL) await #expect(throws: WorkspaceError.self) { _ = try await store.snapshotEntry(id: second.snapshotId, workspaceId: id, path: .root, includeChildren: true) @@ -451,7 +393,7 @@ struct CheckpointFormatTests { ]))) } let store = FileCheckpointStore(rootDirectory: root) - // Independent saved roots exercise persistence without the legacy snapshot + // Independent saved roots exercise persistence without the materialized snapshot // diff's String-keyed map, which is fixed by the separate security track. let first = try await store.saveRevision(snapshot(Data([0, 2])), draft: draft(id)) let second = try await store.saveRevision(snapshot(Data([0, 2])), draft: draft(id)) @@ -561,29 +503,4 @@ struct CheckpointFormatTests { CheckpointDraft(workspaceID: id, label: nil, preferredParentID: parent, origin: .manual) } - // Verbatim cad29c4 manifest types; only the outer type name differs. - private struct V1SnapshotManifest: Codable, Sendable { - struct Node: Codable, Sendable { - enum Kind: String, Codable { - case file, directory, symlink, missing - } - - var kind: Kind - var path: WorkspacePath - var permissions: POSIXPermissions? - var contentHash: String? - var contentSize: UInt64? = nil - var modificationDate: Date? = nil - var symlinkTarget: String? - var children: [Node]? - } - - var version: Int - var id: UUID - var rootPath: WorkspacePath - var root: Node - // Older manifests and materialized snapshots have no trustworthy capture timestamp. - var captureStartedAt: Date? = nil - } - } diff --git a/Tests/WorkspaceTests/CheckpointStoreTests.swift b/Tests/WorkspaceTests/CheckpointStoreTests.swift index 87ad82b..5b98bc0 100644 --- a/Tests/WorkspaceTests/CheckpointStoreTests.swift +++ b/Tests/WorkspaceTests/CheckpointStoreTests.swift @@ -34,8 +34,8 @@ struct CheckpointStoreTests { #expect(other.sequence == 1) } - @Test(arguments: [false, true]) - func `legacy sequence sidecars and reserved torn appends preserve the cursor`(legacySidecar: Bool) async throws { + @Test + func `reserved torn appends preserve the cursor`() async throws { let root = try TestSupport.temporaryDirectory("SequenceSidecar") defer { TestSupport.remove(root) } let id = UUID() @@ -43,8 +43,8 @@ struct CheckpointStoreTests { try await store.appendMutation(Self.sampleMutation(workspaceID: id, path: "/a")) let directory = root.appendingPathComponent(id.uuidString) let sidecar = directory.appendingPathComponent("mutations.sequence.json") - // Simulate a reserved sequence whose append failed mid-UTF8; old stores wrote an Int. - let state = legacySidecar ? "7" : #"{"highWaterMark":7,"retractionEpoch":3}"# + // Simulate a reserved sequence whose append failed mid-UTF8. + let state = #"{"highWaterMark":7,"retractionEpoch":3}"# try Data(state.utf8).write(to: sidecar) let log = directory.appendingPathComponent("mutations.jsonl") let handle = try FileHandle(forWritingTo: log) @@ -57,7 +57,7 @@ struct CheckpointStoreTests { try await store.removeMutation(sequence: 1, workspaceId: id) let page = try await store.listMutations(workspaceId: id, afterSequence: 0) #expect(page.records.isEmpty) - #expect(page.retractionEpoch == (legacySidecar ? 1 : 4)) + #expect(page.retractionEpoch == 4) let next = try await FileCheckpointStore(rootDirectory: root).appendMutation(Self.sampleMutation(workspaceID: id, path: "/next")) #expect(next.sequence == 8) } @@ -200,19 +200,34 @@ struct CheckpointStoreTests { } } - @Test - func `unsupported store versions are rejected`() async throws { - let root = try TestSupport.temporaryDirectory("StoreVersion") + @Test(arguments: [nil, "{}", "not JSON", #"{"version":1}"#, + #"{"format":"other","version":1}"#, + #"{"format":"workspace-history","version":0}"#, + #"{"format":"workspace-history","version":2}"#] as [String?]) + func `invalid store markers require deleting the history directory`(marker: String?) async throws { + let root = try TestSupport.temporaryDirectory("StoreFormat") defer { TestSupport.remove(root) } let workspaceID = UUID() let directory = root.appendingPathComponent(workspaceID.uuidString, isDirectory: true) try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try Data("{\"version\":999}".utf8).write(to: directory.appendingPathComponent("format.json")) + let format = directory.appendingPathComponent("format.json") + if let marker { try Data(marker.utf8).write(to: format) } let store = FileCheckpointStore(rootDirectory: root) - await #expect(throws: WorkspaceError.self) { - _ = try await store.listMutations(workspaceId: workspaceID) + // Neither opening for writes nor reading may silently initialize an existing directory. + for writing in [false, true] { + do { + if writing { try await store.prepare(workspaceId: workspaceID) } + else { _ = try await store.listMutations(workspaceId: workspaceID) } + Issue.record("invalid store marker was accepted") + } catch let error as WorkspaceError { + guard case let .storageCorrupted(message) = error else { throw error } + #expect(message.contains("delete the history directory")) + #expect(message.contains(directory.path)) + } } + #expect(try FileManager.default.contentsOfDirectory(atPath: directory.path) == (marker == nil ? [] : ["format.json"])) + if let marker { #expect(try String(contentsOf: format, encoding: .utf8) == marker) } } private static func sampleSnapshot() -> Snapshot { diff --git a/Tests/WorkspaceTests/EditPathTests.swift b/Tests/WorkspaceTests/EditPathTests.swift index 0909c47..295a515 100644 --- a/Tests/WorkspaceTests/EditPathTests.swift +++ b/Tests/WorkspaceTests/EditPathTests.swift @@ -177,7 +177,7 @@ struct EditPathTests { try await fs.createDirectory(path: "/dir", recursive: true) try await fs.createSymlink(path: "/alias", target: "dir") let workspace = Workspace(fileSystem: fs) - let events = await workspace.watchChanges() + let events = await workspace.events(.init(path: .root, includeCheckpoints: false)) let changes = try await workspace.writeText("/alias/new", "new") #expect(changes.touchedPaths == ["/dir/new"]) #expect(await nextEditEvent(in: events) == .changes(changes)) diff --git a/Tests/WorkspaceTests/FileSystemRegressionTests.swift b/Tests/WorkspaceTests/FileSystemRegressionTests.swift index fbac3e0..3f1afd2 100644 --- a/Tests/WorkspaceTests/FileSystemRegressionTests.swift +++ b/Tests/WorkspaceTests/FileSystemRegressionTests.swift @@ -174,16 +174,6 @@ struct FileSystemRegressionTests { #expect(try await memory.stat(path: "/file").identity != original.identity) } - @Test - func `FileInfo decodes older metadata without identity or link count`() throws { - let legacy = Data(#"{"path":"/file","kind":"file","size":4,"permissions":{"rawValue":420}}"#.utf8) - let info = try JSONDecoder().decode(FileInfo.self, from: legacy) - #expect(info.path == "/file") - #expect(info.size == 4) - #expect(info.identity == nil) - #expect(info.linkCount == nil) - } - @Test func `preexisting disk hard links charge every path and replacement breaks sharing`() async throws { let root = try TestSupport.temporaryDirectory("LimitedDiskAliases") diff --git a/Tests/WorkspaceTests/FilesystemTests.swift b/Tests/WorkspaceTests/FilesystemTests.swift index e5c723b..0d7da9d 100644 --- a/Tests/WorkspaceTests/FilesystemTests.swift +++ b/Tests/WorkspaceTests/FilesystemTests.swift @@ -971,7 +971,7 @@ struct FilesystemTests { let root = try FilesystemTestSupport.makeTempDirectory(prefix: "SandboxURL") defer { FilesystemTestSupport.removeDirectory(root) } - let filesystem = try SandboxFilesystem(root: .url(root)) + let filesystem = try SandboxFileSystem(root: .url(root)) try await filesystem.writeFile(path: "/note.txt", data: FilesystemTestSupport.data("hello"), append: false) try await filesystem.createDirectory(path: "/dir", recursive: true) @@ -995,18 +995,18 @@ struct FilesystemTests { @Test(.tags(.sandbox)) func `sandbox filesystem supports standard roots configuration and app-group validation`() async throws { - let temporary = try SandboxFilesystem(root: .temporary) + let temporary = try SandboxFileSystem(root: .temporary) #expect(await temporary.exists(path: "/")) #if os(macOS) - let documents = try SandboxFilesystem(root: .documents) - let caches = try SandboxFilesystem(root: .caches) + let documents = try SandboxFileSystem(root: .documents) + let caches = try SandboxFileSystem(root: .caches) #expect(await documents.exists(path: "/")) #expect(await caches.exists(path: "/")) #endif do { - _ = try SandboxFilesystem(root: .appGroup("invalid-group")) + _ = try SandboxFileSystem(root: .appGroup("invalid-group")) Issue.record("expected invalid app group identifier rejection") } catch let error as WorkspaceError { #expect(error.description.contains("invalid app group identifier")) @@ -1014,7 +1014,7 @@ struct FilesystemTests { #if canImport(Darwin) do { - _ = try SandboxFilesystem( + _ = try SandboxFileSystem( root: .appGroup("group.workspace.tests.missing"), fileManager: NilAppGroupFileManager() ) @@ -1024,7 +1024,7 @@ struct FilesystemTests { } #else do { - _ = try SandboxFilesystem(root: .appGroup("group.workspace.tests.missing")) + _ = try SandboxFileSystem(root: .appGroup("group.workspace.tests.missing")) Issue.record("expected unavailable app group rejection") } catch let error as WorkspaceError { #expect(error.description.contains("app group container unavailable")) @@ -1037,7 +1037,7 @@ struct FilesystemTests { let secondRoot = try FilesystemTestSupport.makeTempDirectory(prefix: "SandboxSecond") defer { FilesystemTestSupport.removeDirectory(secondRoot) } - let filesystem = try SandboxFilesystem(root: .url(firstRoot)) + let filesystem = try SandboxFileSystem(root: .url(firstRoot)) try await filesystem.writeFile(path: "/configured.txt", data: FilesystemTestSupport.data("configured"), append: false) #expect(FileManager.default.fileExists(atPath: firstRoot.appendingPathComponent("configured.txt").path)) @@ -1082,7 +1082,7 @@ struct FilesystemTests { let secondRoot = try FilesystemTestSupport.makeTempDirectory(prefix: "SecurityScopedSecond") defer { FilesystemTestSupport.removeDirectory(secondRoot) } - let filesystem = try SecurityScopedFilesystem(url: firstRoot, mode: .readWrite) + let filesystem = try SecurityScopedFileSystem(url: firstRoot, mode: .readWrite) try await filesystem.writeFile(path: "/note.txt", data: FilesystemTestSupport.data("hello"), append: false) try await filesystem.createDirectory(path: "/dir", recursive: true) @@ -1109,7 +1109,7 @@ struct FilesystemTests { try await filesystem.writeFile(path: "/fresh.txt", data: FilesystemTestSupport.data("fresh"), append: false) #expect(FileManager.default.fileExists(atPath: secondRoot.appendingPathComponent("fresh.txt").path)) - let readOnly = try SecurityScopedFilesystem(url: secondRoot, mode: .readOnly) + let readOnly = try SecurityScopedFileSystem(url: secondRoot, mode: .readOnly) #expect(try await readOnly.readFile(path: "/fresh.txt") == FilesystemTestSupport.data("fresh")) do { @@ -1128,7 +1128,7 @@ struct FilesystemTests { let store = UserDefaultsBookmarkStore(suiteName: suiteName, keyPrefix: "workspace.tests.") do { - _ = try await SecurityScopedFilesystem.loadBookmark(id: "missing", store: store) + _ = try await SecurityScopedFileSystem.loadBookmark(id: "missing", store: store) Issue.record("expected missing bookmark rejection") } catch let error as WorkspaceError { #expect(error.description.contains("bookmark not found")) @@ -1144,7 +1144,7 @@ struct FilesystemTests { try await store.saveBookmark(FilesystemTestSupport.data("not-a-bookmark"), for: "invalid") do { - _ = try await SecurityScopedFilesystem.loadBookmark(id: "invalid", store: store) + _ = try await SecurityScopedFileSystem.loadBookmark(id: "invalid", store: store) Issue.record("expected invalid bookmark rejection") } catch let error as NSError { #expect(error.domain == NSCocoaErrorDomain) @@ -1161,7 +1161,7 @@ struct FilesystemTests { let suiteName = FilesystemTestSupport.uniqueSuiteName() defer { UserDefaults(suiteName: suiteName)?.removePersistentDomain(forName: suiteName) } - let filesystem = try SecurityScopedFilesystem(url: root, mode: .readWrite) + let filesystem = try SecurityScopedFileSystem(url: root, mode: .readWrite) let store = UserDefaultsBookmarkStore(suiteName: suiteName, keyPrefix: "workspace.tests.") do { diff --git a/Tests/WorkspaceTests/Fixtures/README.md b/Tests/WorkspaceTests/Fixtures/README.md deleted file mode 100644 index 0f55119..0000000 --- a/Tests/WorkspaceTests/Fixtures/README.md +++ /dev/null @@ -1,7 +0,0 @@ -`v1/` was generated using the original `FileCheckpointStore` from commit -`cad29c479e28ce905d4e10cf3675c59838c77928`, before changing the storage format. -It contains two checkpoints of `/note`, `/nested/deep/file`, and a relative symlink -`/link -> nested/deep/file`, an append-only mutation log, and the -`mutations.sequence.json` high-water-mark/retraction-epoch sidecar. The first -checkpoint is labeled `legacy`; the second is unlabeled. The bytes are copied -verbatim so compatibility tests exercise persisted data from the old writer. diff --git a/Tests/WorkspaceTests/Fixtures/v1/.10000000-0000-0000-0000-000000000001.format.lock b/Tests/WorkspaceTests/Fixtures/v1/.10000000-0000-0000-0000-000000000001.format.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 deleted file mode 100644 index 0ad6ad6..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47 +++ /dev/null @@ -1 +0,0 @@ -nested legacy diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 deleted file mode 100644 index a8237ec..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9 +++ /dev/null @@ -1 +0,0 @@ -legacy checkpoint diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 deleted file mode 100644 index b41fb5d..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/blobs/646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2 +++ /dev/null @@ -1 +0,0 @@ -legacy updated diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json deleted file mode 100644 index fbbc724..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/669D476B-6EFC-46E0-A1FB-8155C3DA8121.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "createdAt" : 814174270.20685, - "firstMutationSequence" : 1, - "id" : "669D476B-6EFC-46E0-A1FB-8155C3DA8121", - "label" : "legacy", - "lastMutationSequence" : 1, - "mutationCursor" : 1, - "origin" : { - "manual" : { - - } - }, - "snapshotId" : "5D7DC62F-9749-41E4-8BCE-90306A25C30C", - "summary" : { - "changedPathCount" : 6, - "hasTextChanges" : true, - "statistics" : { - "additions" : 2, - "changedFileCount" : 2, - "changedPathCount" : 6, - "deletions" : 0 - }, - "touchedPaths" : [ - "\/", - "\/link", - "\/nested", - "\/nested\/deep", - "\/nested\/deep\/file", - "\/note" - ] - }, - "workspaceID" : "10000000-0000-0000-0000-000000000001" -} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json deleted file mode 100644 index 1c645cd..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/checkpoints/8BB89064-AA2F-4516-BCE8-5D6D2FE4E092.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "createdAt" : 814174270.213568, - "firstMutationSequence" : 2, - "id" : "8BB89064-AA2F-4516-BCE8-5D6D2FE4E092", - "lastMutationSequence" : 2, - "mutationCursor" : 2, - "origin" : { - "manual" : { - - } - }, - "parentID" : "669D476B-6EFC-46E0-A1FB-8155C3DA8121", - "snapshotId" : "EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76", - "summary" : { - "changedPathCount" : 1, - "hasTextChanges" : true, - "statistics" : { - "additions" : 1, - "changedFileCount" : 1, - "changedPathCount" : 1, - "deletions" : 1 - }, - "touchedPaths" : [ - "\/note" - ] - }, - "workspaceID" : "10000000-0000-0000-0000-000000000001" -} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json deleted file mode 100644 index 2da5380..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/format.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "version" : 1 -} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/lifecycle.lock b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/lifecycle.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl deleted file mode 100644 index 2855807..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"operation":"edit","changes":{"changes":[],"textDiffOmissions":[]},"workspaceID":"10000000-0000-0000-0000-000000000001","timestamp":814174270.19686,"sequence":1} -{"workspaceID":"10000000-0000-0000-0000-000000000001","changes":{"changes":[],"textDiffOmissions":[]},"timestamp":814174270.210703,"operation":"edit","sequence":2} diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.lock b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.lock deleted file mode 100644 index e69de29..0000000 diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json deleted file mode 100644 index e29520b..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/mutations.sequence.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "highWaterMark" : 2, - "retractionEpoch" : 0 -} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json deleted file mode 100644 index 3532972..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/5D7DC62F-9749-41E4-8BCE-90306A25C30C.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "captureStartedAt" : 814174270.205489, - "id" : "5D7DC62F-9749-41E4-8BCE-90306A25C30C", - "root" : { - "children" : [ - { - "kind" : "symlink", - "modificationDate" : 814174270.196814, - "path" : "\/link", - "permissions" : { - "rawValue" : 511 - }, - "symlinkTarget" : "nested\/deep\/file" - }, - { - "children" : [ - { - "children" : [ - { - "contentHash" : "3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47", - "contentSize" : 14, - "kind" : "file", - "modificationDate" : 814174270.196806, - "path" : "\/nested\/deep\/file", - "permissions" : { - "rawValue" : 420 - } - } - ], - "kind" : "directory", - "modificationDate" : 814174270.196807, - "path" : "\/nested\/deep", - "permissions" : { - "rawValue" : 493 - } - } - ], - "kind" : "directory", - "modificationDate" : 814174270.196769, - "path" : "\/nested", - "permissions" : { - "rawValue" : 493 - } - }, - { - "contentHash" : "47ae1bb0b991086ce0f299d281927e7ef4c37aedfe57f6d8f22862833e8b98e9", - "contentSize" : 18, - "kind" : "file", - "modificationDate" : 814174270.196821, - "path" : "\/note", - "permissions" : { - "rawValue" : 420 - } - } - ], - "kind" : "directory", - "modificationDate" : 814174270.196822, - "path" : "\/", - "permissions" : { - "rawValue" : 493 - } - }, - "rootPath" : "\/", - "version" : 2 -} \ No newline at end of file diff --git a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json b/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json deleted file mode 100644 index 48515db..0000000 --- a/Tests/WorkspaceTests/Fixtures/v1/10000000-0000-0000-0000-000000000001/snapshots/EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "captureStartedAt" : 814174270.212788, - "id" : "EC4895B2-725C-4EDF-B0B9-E22C4B6F6C76", - "root" : { - "children" : [ - { - "kind" : "symlink", - "modificationDate" : 814174270.196814, - "path" : "\/link", - "permissions" : { - "rawValue" : 511 - }, - "symlinkTarget" : "nested\/deep\/file" - }, - { - "children" : [ - { - "children" : [ - { - "contentHash" : "3b27679a3f38b27d960362cf6f08d7743c58b19cd10be7ec2666cc5a006f5c47", - "contentSize" : 14, - "kind" : "file", - "modificationDate" : 814174270.196806, - "path" : "\/nested\/deep\/file", - "permissions" : { - "rawValue" : 420 - } - } - ], - "kind" : "directory", - "modificationDate" : 814174270.196807, - "path" : "\/nested\/deep", - "permissions" : { - "rawValue" : 493 - } - } - ], - "kind" : "directory", - "modificationDate" : 814174270.196769, - "path" : "\/nested", - "permissions" : { - "rawValue" : 493 - } - }, - { - "contentHash" : "646d3b4ced4632ae3ed243aff9b70ad0b84a09286df071b9159796462b635ae2", - "contentSize" : 15, - "kind" : "file", - "modificationDate" : 814174270.2107, - "path" : "\/note", - "permissions" : { - "rawValue" : 420 - } - } - ], - "kind" : "directory", - "modificationDate" : 814174270.210701, - "path" : "\/", - "permissions" : { - "rawValue" : 493 - } - }, - "rootPath" : "\/", - "version" : 2 -} \ No newline at end of file diff --git a/Tests/WorkspaceTests/LimitedCacheTests.swift b/Tests/WorkspaceTests/LimitedCacheTests.swift index 6e04adb..e2de16c 100644 --- a/Tests/WorkspaceTests/LimitedCacheTests.swift +++ b/Tests/WorkspaceTests/LimitedCacheTests.swift @@ -456,7 +456,6 @@ private actor LimitedCacheBackend: CompleteFileSystemChangeSource, PotentialPath if subscriptionFails { throw CacheTestError.injected } return stream } - func changes() async throws -> AsyncStream { unscopedChanges(try await changeSignals()) } func stat(path: WorkspacePath) async throws -> FileInfo { statted.append(path) if path.dirname.isRoot, path.basename.hasPrefix("fixed-"), diff --git a/Tests/WorkspaceTests/ReviewRevisionTests.swift b/Tests/WorkspaceTests/ReviewRevisionTests.swift index 5fa8669..256da13 100644 --- a/Tests/WorkspaceTests/ReviewRevisionTests.swift +++ b/Tests/WorkspaceTests/ReviewRevisionTests.swift @@ -79,7 +79,7 @@ struct ReviewRevisionTests { let checkpoint = try await ws.createCheckpoint() // Simulate a newer revision arriving through the store without re-seeding this instance. _ = try await ws.store.saveRevision(Snapshot.capture(from: fs.base), draft: .init( - workspaceID: checkpoint.workspaceId, label: nil, preferredParentID: checkpoint.id, origin: .manual)) + workspaceID: checkpoint.workspaceID, label: nil, preferredParentID: checkpoint.id, origin: .manual)) let reads = await fs.reads.count _ = try await ws.compact(retaining: .latest(1, preservingLabeled: false), dryRun: true) try await ws.beginTransaction().discard() diff --git a/Tests/WorkspaceTests/SearchEquivalenceTests.swift b/Tests/WorkspaceTests/SearchEquivalenceTests.swift index 33e7285..16f938d 100644 --- a/Tests/WorkspaceTests/SearchEquivalenceTests.swift +++ b/Tests/WorkspaceTests/SearchEquivalenceTests.swift @@ -82,7 +82,7 @@ struct SearchEquivalenceTests { } // Frozen pre-optimization algorithm from cad29c4. This intentionally uses - // independent line Strings and Foundation matching as the compatibility oracle, + // independent line Strings and Foundation matching as the reference implementation, // including pagination, byte budgets, skipped files and context serialization. static func referenceSearch(_ request: SearchRequest, on fileSystem: any FileSystem) async throws -> SearchResult { guard request.contextLines >= 0, request.limits.maxFiles >= 0, diff --git a/Tests/WorkspaceTests/UnicodePathRegressionTests.swift b/Tests/WorkspaceTests/UnicodePathRegressionTests.swift index 97f5e2c..363eae1 100644 --- a/Tests/WorkspaceTests/UnicodePathRegressionTests.swift +++ b/Tests/WorkspaceTests/UnicodePathRegressionTests.swift @@ -228,7 +228,7 @@ struct UnicodePathRegressionTests { func recursiveEventsIncludeUnicodeDescendants() async throws { let workspace = Workspace() try await workspace.createDirectory("/d") - let stream = await workspace.watchChanges(at: "/d") + let stream = await workspace.events(.init(path: "/d", includeCheckpoints: false)) let expected = try await workspace.writeText("/d/\u{301}x", "first") // A second matching event makes an omitted first event observable without // timing assumptions or an indefinitely suspended iterator on the baseline. diff --git a/Tests/WorkspaceTests/WorkspaceAPITests.swift b/Tests/WorkspaceTests/WorkspaceAPITests.swift index 3c77e39..10e5f70 100644 --- a/Tests/WorkspaceTests/WorkspaceAPITests.swift +++ b/Tests/WorkspaceTests/WorkspaceAPITests.swift @@ -16,12 +16,12 @@ struct WorkspaceAPITests { } @Test - func `external local filesystem mutations reach watchChanges without entering history`() async throws { + func `external local filesystem mutations reach events without entering history`() async throws { let root = try TestSupport.temporaryDirectory("ExternalEvents") defer { TestSupport.remove(root) } let files = try LocalFileSystem(root: root) let workspace = Workspace(fileSystem: files) - let stream = await workspace.watchChanges() + let stream = await workspace.events(.init(path: .root, includeCheckpoints: false)) try await files.writeFile(path: "/external.txt", data: Data("outside".utf8), append: false) @@ -38,7 +38,7 @@ struct WorkspaceAPITests { let root = try TestSupport.temporaryDirectory("EventDeduplication") defer { TestSupport.remove(root) } let workspace = Workspace(fileSystem: try LocalFileSystem(root: root)) - let stream = await workspace.watchChanges() + let stream = await workspace.events(.init(path: .root, includeCheckpoints: false)) let expected = try await workspace.writeText("/note", "once") From 08fbf81dbdeae4df38dda761f07772e2be86dc4f Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 09:16:40 -0700 Subject: [PATCH 25/34] Publish new history directories atomically and tidy capture edges A new workspace history directory is now staged with its synchronized format marker and renamed into place, followed by a sync of the parent directory, so a crash during first open leaves only disposable staging instead of an unmarked directory that later opens reject. Materialized saves drop .missing children the same way capture does, two unused persistence helpers are removed, and the capture-route parity test pins in-memory timestamps so its racy-window expectations no longer depend on scheduling. --- Sources/Workspace/CheckpointStore.swift | 20 ++++-- Sources/Workspace/Workspace+Internals.swift | 34 ---------- .../CheckpointCaptureTraversalTests.swift | 63 +++++++++++++++---- .../WorkspaceTests/CheckpointStoreTests.swift | 52 +++++++++++++++ 4 files changed, 118 insertions(+), 51 deletions(-) diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index f73b93a..8cdb4a7 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -996,8 +996,9 @@ actor FileCheckpointStore: CheckpointStore { case let .directory(directory): let old = try previous.map { try children(of: $0, workspaceId: workspaceId) } ?? [] let byPath = Dictionary(uniqueKeysWithValues: old.map { ($0.path, $0) }) - let entries = try directory.children.map { - try writeManifestNode($0, workspaceId: workspaceId, batch: batch, reusing: byPath[$0.path]) + let entries = try directory.children.compactMap { child -> SnapshotManifest.Node? in + if case .missing = child { return nil } + return try writeManifestNode(child, workspaceId: workspaceId, batch: batch, reusing: byPath[child.path]) }.sorted { PathName($0.path.name) < PathName($1.path.name) } if let previous, previous.kind == .directory, previous.treeHash != nil, old == entries { var reused = previous @@ -1729,13 +1730,24 @@ actor FileCheckpointStore: CheckpointStore { private func ensureWorkspaceDirectoriesLocked(for workspaceId: UUID) throws { let workspaceURL = workspaceDirectoryURL(workspaceId: workspaceId) + // This workspace's format lock also owns its staging directory. Never sweep + // another workspace's staging: its first open may still be in progress. + let stagingURL = rootDirectory.appendingPathComponent(".\(workspaceId.uuidString).pending", isDirectory: true) + try removeIfPresent(stagingURL) let existed = fileManager.fileExists(atPath: workspaceURL.path) if existed { try validateWorkspaceFormatLocked(workspaceId) } else { - try ensureDirectory(workspaceURL) + try ensureDirectory(stagingURL) try Self.durableWrite(encoder.encode(StoreFormat()), - to: workspaceURL.appendingPathComponent("format.json")) + to: stagingURL.appendingPathComponent("format.json")) + // Both paths have the same parent, so publication is an atomic rename. + // A crash before it leaves only disposable staging; after it, the + // published directory always contains a complete, synchronized marker. + try fileManager.moveItem(at: stagingURL, to: workspaceURL) + // Persist the rename before anything is written inside the published + // directory; otherwise a crash could resurrect it as sweepable staging. + try Self.syncDirectory(rootDirectory) } // A prior initialization may have stopped after publishing the marker. let newObjects = !fileManager.fileExists(atPath: treesDirectoryURL(workspaceId: workspaceId).path) diff --git a/Sources/Workspace/Workspace+Internals.swift b/Sources/Workspace/Workspace+Internals.swift index dfe5031..527a5dc 100644 --- a/Sources/Workspace/Workspace+Internals.swift +++ b/Sources/Workspace/Workspace+Internals.swift @@ -89,29 +89,6 @@ extension Workspace { return snapshot } - func persistCheckpoint( - snapshot: Snapshot, - label: String?, - parentCheckpointId: UUID?, - origin: Checkpoint.Origin - ) async throws -> Checkpoint { - let checkpoint = try await store.saveRevision( - snapshot, - draft: CheckpointDraft( - workspaceID: workspaceId, - label: label, - preferredParentID: parentCheckpointId, - origin: origin - ) - ) - checkpoints.removeAll { $0.id == checkpoint.id } - checkpoints.append(checkpoint) - checkpoints.sort(by: checkpointSort) - headCheckpointId = Checkpoint.lineageHeadID(in: checkpoints) - emitWorkspaceEvent(.checkpoint(checkpoint)) - return checkpoint - } - @discardableResult func appendMutation(operation: Mutation.Operation, changes: ChangeSet) async throws -> Mutation? { guard recording != .off else { return nil } @@ -125,17 +102,6 @@ extension Workspace { return persisted } - func persistMutationCheckpoint( - snapshot: Snapshot, label: String?, origin: Checkpoint.Origin, - operation: Mutation.Operation, changes: ChangeSet - ) async throws -> Checkpoint { - try await withMutationCheckpoint(operation: operation, changes: changes) { - try await persistCheckpoint( - snapshot: snapshot, label: label, parentCheckpointId: headCheckpointId, origin: origin - ) - } - } - // If checkpoint persistence fails after the history append, retract exactly // our record. Other instances' mutations must not be removed by compensation. func withMutationCheckpoint( diff --git a/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift b/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift index d8a9304..a438fb3 100644 --- a/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift +++ b/Tests/WorkspaceTests/CheckpointCaptureTraversalTests.swift @@ -38,8 +38,6 @@ struct CheckpointCaptureTraversalTests { try await unicode.createFile(path: unicodeRoot.appending("cafe\u{301}"), data: Data([2])) let source: any FileSystem = inMemory ? local : MountedFileSystem(base: local, mounts: [.init(mountPoint: "/unicode", fileSystem: unicode)]) - let bulk = CaptureSource(source, scoped: true) - let paths = CaptureSource(source, scoped: false) let id = UUID() let roots = [container.appendingPathComponent("bulk"), container.appendingPathComponent("paths")] let stores = roots.map { FileCheckpointStore(rootDirectory: $0, checkpointCapture: policy) } @@ -60,8 +58,20 @@ struct CheckpointCaptureTraversalTests { try await local.remove(path: "/alias", recursive: false) try await local.createSymlink(path: "/alias", target: "missing") } - bulk.reset() - paths.reset() + // Pin in-memory mtimes well outside the racy window on both routes, + // including the mounted in-memory subtree in the local variant. + // Advance only the rewritten file's timestamp so metadata reuse must + // still detect its same-size content change in phase 2. + let transformInfo: @Sendable (FileInfo) -> FileInfo = { info in + var info = info + if info.kind == .file, inMemory || info.path.isWithin("/unicode") { + let changed = phase >= 2 && info.path == "/unicode/cafe\u{301}" + info.modificationDate = old.addingTimeInterval(changed ? 1 : 0) + } + return info + } + let bulk = CaptureSource(source, scoped: true, transformInfo: transformInfo) + let paths = CaptureSource(source, scoped: false, transformInfo: transformInfo) var manifests: [Data] = [] var summaries: [ChangeSet.Summary] = [] for (i, fs) in [bulk, paths].enumerated() { @@ -75,7 +85,7 @@ struct CheckpointCaptureTraversalTests { for (path, entry) in expected.entries { let hint = try #require(captured.metadata.entries[path]) #expect(hint.info.identity == entry.info.identity) - #expect(hint.info.modificationDate == entry.info.modificationDate) + #expect(hint.info.modificationDate == transformInfo(entry.info).modificationDate) #expect(hint.info.permissions == entry.info.permissions) #expect(hint.info.kind == entry.info.kind) #expect(hint.target == entry.target) @@ -101,15 +111,37 @@ struct CheckpointCaptureTraversalTests { #expect(bulk.walkCount == 1) #expect(bulk.statCount == 0 && bulk.listCount == 0) #expect(bulk.reads == paths.reads) - if phase == 1, policy == .trustMetadata, !inMemory { - #expect(!bulk.reads.contains("/a/file")) - #expect(!bulk.reads.contains("/hard")) - #expect(!bulk.reads.contains("/a-")) + if policy == .trustMetadata { + if phase == 1 { #expect(bulk.reads.isEmpty) } + if phase == 2 { #expect(bulk.reads == ["/a/new", "/unicode/cafe\u{301}"]) } } if policy == .verifyContents { #expect(bulk.reads.contains("/a/file")) } } } + @Test + func materializedSaveOmitsChildrenThatVanishedDuringCapture() async throws { + let root = try TestSupport.temporaryDirectory("VanishedSnapshotChild") + defer { TestSupport.remove(root) } + let base = InMemoryFileSystem() + try await base.writeFile(path: "/present", data: Data("retained".utf8), append: false) + let snapshot = try await Snapshot.capture(from: GhostFileSystem(base: base, code: ENOENT)) + guard case let .directory(directory) = snapshot.entry else { + Issue.record("expected a directory snapshot") + return + } + #expect(directory.children.contains { if case .missing = $0 { return true }; return false }) + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + let checkpoint = try await store.saveRevision(snapshot, + draft: .init(workspaceID: id, label: nil, preferredParentID: nil, origin: .manual)) + let reopened = FileCheckpointStore(rootDirectory: root) + let saved = try #require(await reopened.loadSnapshot(id: checkpoint.snapshotId, workspaceId: id)) + #expect(saved.entry == (try await Snapshot.capture(from: base)).entry) + #expect(try await reopened.snapshotEntry(id: checkpoint.snapshotId, workspaceId: id, + path: "/ghost", includeChildren: false) == nil) + } + @Test(arguments: [false, true]) func racyTimestampsAndMissingBlobsForceReads(scoped: Bool) async throws { let container = try TestSupport.temporaryDirectory("CaptureReuse") @@ -297,6 +329,7 @@ private final class CaptureSource: FileSystem, TraversingFileSystem, ExactTimest let base: any FileSystem let supportsScopedWalk: Bool let failingRead: Bool + private let transformInfo: @Sendable (FileInfo) -> FileInfo private let lock = NSLock() private var recordedReads: [WorkspacePath] = [] private var counts = (walk: 0, stat: 0, list: 0) @@ -304,8 +337,10 @@ private final class CaptureSource: FileSystem, TraversingFileSystem, ExactTimest var walkCount: Int { lock.withLock { counts.walk } } var statCount: Int { lock.withLock { counts.stat } } var listCount: Int { lock.withLock { counts.list } } - init(_ base: any FileSystem, scoped: Bool, failingRead: Bool = false) { + init(_ base: any FileSystem, scoped: Bool, failingRead: Bool = false, + transformInfo: @escaping @Sendable (FileInfo) -> FileInfo = { $0 }) { self.base = base; supportsScopedWalk = scoped; self.failingRead = failingRead + self.transformInfo = transformInfo } func reset() { lock.withLock { recordedReads = []; counts = (0, 0, 0) } } private func record(_ path: WorkspacePath) throws { @@ -319,7 +354,7 @@ private final class CaptureSource: FileSystem, TraversingFileSystem, ExactTimest skippingVanishedEntries: skippingVanishedEntries) { info, read, link in try await withoutActuallyEscaping(read) { read in try await withoutActuallyEscaping(link) { link in - try await visit(info, { length in + try await visit(self.transformInfo(info), { length in try self.record(info.path) return try await read(length) }, { try await link() }) @@ -329,11 +364,13 @@ private final class CaptureSource: FileSystem, TraversingFileSystem, ExactTimest } func stat(path: WorkspacePath) async throws -> FileInfo { lock.withLock { counts.stat += 1 } - return try await base.stat(path: path) + return transformInfo(try await base.stat(path: path)) } func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { lock.withLock { counts.list += 1 } - return try await base.listDirectory(path: path) + return try await base.listDirectory(path: path).map { + DirectoryEntry(name: $0.name, info: transformInfo($0.info)) + } } func readFile(path: WorkspacePath) async throws -> Data { try record(path); return try await base.readFile(path: path) } func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } diff --git a/Tests/WorkspaceTests/CheckpointStoreTests.swift b/Tests/WorkspaceTests/CheckpointStoreTests.swift index 5b98bc0..d7b3705 100644 --- a/Tests/WorkspaceTests/CheckpointStoreTests.swift +++ b/Tests/WorkspaceTests/CheckpointStoreTests.swift @@ -200,6 +200,58 @@ struct CheckpointStoreTests { } } + @Test(arguments: ["empty directory", "temporary marker", "completed marker"]) + func `interrupted first open leaves only recoverable staging`(crashPoint: String) async throws { + let root = try TestSupport.temporaryDirectory("StoreFirstOpen") + defer { TestSupport.remove(root) } + let id = UUID() + let staging = root.appendingPathComponent(".\(id.uuidString).pending", isDirectory: true) + let published = root.appendingPathComponent(id.uuidString, isDirectory: true) + // Another workspace may be initializing under its own lock. + let otherStaging = root.appendingPathComponent(".\(UUID().uuidString).pending", isDirectory: true) + try FileManager.default.createDirectory(at: otherStaging, withIntermediateDirectories: false) + let interrupted = FileCheckpointStore(rootDirectory: root, + fileManager: FirstOpenCrashManager(staging: staging, crashPoint: crashPoint)) + await #expect(throws: TestFileSystemError.self) { try await interrupted.prepare(workspaceId: id) } + #expect(!FileManager.default.fileExists(atPath: published.path)) + #expect(FileManager.default.fileExists(atPath: staging.path)) + if crashPoint == "completed marker" { + #expect(FileManager.default.fileExists(atPath: staging.appendingPathComponent("format.json").path)) + } + + let recovered = FileCheckpointStore(rootDirectory: root) + let snapshot = Self.sampleSnapshot() + let checkpoint = try await recovered.saveRevision(snapshot, draft: Self.sampleDraft(workspaceID: id)) + #expect(!FileManager.default.fileExists(atPath: staging.path)) + #expect(FileManager.default.fileExists(atPath: otherStaging.path)) + let reopened = FileCheckpointStore(rootDirectory: root) + #expect(try await reopened.loadSnapshot(id: checkpoint.snapshotId, workspaceId: id) == snapshot) + } + + private final class FirstOpenCrashManager: FileManager, @unchecked Sendable { + let staging: URL + let crashPoint: String + init(staging: URL, crashPoint: String) { + self.staging = staging + self.crashPoint = crashPoint + super.init() + } + override func createDirectory(at url: URL, withIntermediateDirectories createIntermediates: Bool, + attributes: [FileAttributeKey: Any]? = nil) throws { + try super.createDirectory(at: url, withIntermediateDirectories: createIntermediates, attributes: attributes) + if url == staging, crashPoint != "completed marker" { + if crashPoint == "temporary marker" { + try Data("{partial".utf8).write(to: url.appendingPathComponent(".pending-\(UUID().uuidString)-format.json")) + } + throw TestFileSystemError.injected + } + } + override func moveItem(at source: URL, to destination: URL) throws { + if source == staging { throw TestFileSystemError.injected } + try super.moveItem(at: source, to: destination) + } + } + @Test(arguments: [nil, "{}", "not JSON", #"{"version":1}"#, #"{"format":"other","version":1}"#, #"{"format":"workspace-history","version":0}"#, From 7a6e439283412aef866fb5e764fde5ce818335cf Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 20:54:53 -0700 Subject: [PATCH 26/34] Keep entry equality exact and make the accounting cache cap configurable Checkpoint-store Node and TreeEntry and MetadataIndex.Entry compare exactly again, matching what they encode. Reuse decisions that tolerate device renumbering (directory and tree reuse, compareNodes, publishRevision, compaction relocation, observation) use a named isReusable(as:) instead. LimitedFileSystem takes an accountingCacheEntryLimit initializer parameter: nil keeps the platform default (250,000 entries, 25,000 on iOS and watchOS), and 0 never retains accounting between mutations. --- README.md | 7 ++-- .../Workspace/CheckpointStore+Revision.swift | 2 +- Sources/Workspace/CheckpointStore.swift | 32 ++++++++++++----- Sources/Workspace/FS/LimitedFileSystem.swift | 18 ++++++---- Sources/Workspace/FilesystemObservation.swift | 3 +- Sources/Workspace/MetadataIndex.swift | 27 ++++++++++++-- .../WorkspaceTests/DeviceIdentityTests.swift | 30 ++++++++++++++++ Tests/WorkspaceTests/LimitedCacheTests.swift | 35 ++++++++++++++++--- 8 files changed, 129 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index f09fbde..351624d 100644 --- a/README.md +++ b/README.md @@ -505,8 +505,11 @@ declares complete coverage only when its base and every mount do, including late Incomplete or undeclared coverage uses full walks (or the exact generation capability). Each Limited over Local registers its own recursive inotify watcher on Linux; large trees can exhaust the watch limit (ENOSPC), at which point accounting safely falls back to walks. -The retained cache is capped at 250,000 entries (25,000 on iOS/watchOS); above that cap, -accounting is discarded after the operation and every preflight walks the tree. +`LimitedFileSystem(base:limits:accountingCacheEntryLimit:)` caps retained accounting entries, +including the root. Pass `nil` (the default) for 250,000 entries (25,000 on iOS/watchOS), +or an explicit entry count to tune memory retention. `0` (or a negative value) disables +retention, so every preflight walks the tree. Above the cap, accounting is discarded +after the operation. This bounds retained entries, not peak memory during a walk. **Consistency:** a quota decision incorporates signals processed by this Limited actor before its projection finishes. Signals buffered in the backend or stream, or queued diff --git a/Sources/Workspace/CheckpointStore+Revision.swift b/Sources/Workspace/CheckpointStore+Revision.swift index 5520497..a78b2df 100644 --- a/Sources/Workspace/CheckpointStore+Revision.swift +++ b/Sources/Workspace/CheckpointStore+Revision.swift @@ -152,7 +152,7 @@ extension FileCheckpointStore { permissions: directory.permissions, modificationDate: metadata.entries[directory.path]?.info.modificationDate, identity: metadata.entries[directory.path]?.info.identity) - if let old, old.kind == .directory, old.treeHash != nil, updated == previous { + if let old, old.kind == .directory, old.treeHash != nil, updated.isReusable(as: previous) { node.treeHash = old.treeHash return node } diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 8cdb4a7..a286d5a 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -443,7 +443,7 @@ actor FileCheckpointStore: CheckpointStore { /// A small root with content-addressed directory references and optional pack ranges. struct SnapshotManifest: Codable, Sendable { - struct Node: Codable, Sendable, Equatable { + struct Node: Codable, Sendable, MetadataReusable { enum Kind: String, Codable { case file, directory, symlink, missing } @@ -463,10 +463,18 @@ actor FileCheckpointStore: CheckpointStore { lhs.kind == rhs.kind && lhs.path == rhs.path && lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && - FileIdentity.matchesForMetadataReuse(lhs.identity, rhs.identity) && + lhs.identity == rhs.identity && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } + + /// Compare all persisted fields, allowing only device renumbering in identity. + func isReusable(as other: Self) -> Bool { + guard FileIdentity.matchesForMetadataReuse(identity, other.identity) else { return false } + var comparable = self + comparable.identity = other.identity + return comparable == other + } } var id: UUID @@ -478,7 +486,7 @@ actor FileCheckpointStore: CheckpointStore { } /// Names are validated before entering the cache; no absolute paths are persisted. - private struct TreeEntry: Codable, Sendable, Equatable { + private struct TreeEntry: Codable, Sendable, MetadataReusable { var name: String var kind: SnapshotManifest.Node.Kind var permissions: POSIXPermissions @@ -534,11 +542,19 @@ actor FileCheckpointStore: CheckpointStore { lhs.name.utf8.elementsEqual(rhs.name.utf8) && lhs.kind == rhs.kind && lhs.permissions == rhs.permissions && lhs.contentHash == rhs.contentHash && lhs.contentSize == rhs.contentSize && lhs.modificationDate == rhs.modificationDate && - FileIdentity.matchesForMetadataReuse(lhs.identity, rhs.identity) && + lhs.identity == rhs.identity && lhs.symlinkTarget.map(PathName.init) == rhs.symlinkTarget.map(PathName.init) && lhs.treeHash == rhs.treeHash && lhs.packed == rhs.packed } + /// Compare all persisted fields, allowing only device renumbering in identity. + func isReusable(as other: Self) -> Bool { + guard FileIdentity.matchesForMetadataReuse(identity, other.identity) else { return false } + var comparable = self + comparable.identity = other.identity + return comparable == other + } + /// Trees are validated and sorted by exact name bytes before being cached. static func index(of name: String, in entries: [Self]) -> Int? { var low = 0, high = entries.count @@ -754,7 +770,7 @@ actor FileCheckpointStore: CheckpointStore { ) throws -> SnapshotManifest.Node { let entries = directory.children.sorted { PathName($0.name) < PathName($1.name) } if let previous = directory.previous, previous.kind == .directory, previous.treeHash != nil, - directory.previousEntries == entries { + directory.previousEntries.isReusable(as: entries) { var reused = previous reused.permissions = info.permissions reused.modificationDate = info.modificationDate @@ -1000,7 +1016,7 @@ actor FileCheckpointStore: CheckpointStore { if case .missing = child { return nil } return try writeManifestNode(child, workspaceId: workspaceId, batch: batch, reusing: byPath[child.path]) }.sorted { PathName($0.path.name) < PathName($1.path.name) } - if let previous, previous.kind == .directory, previous.treeHash != nil, old == entries { + if let previous, previous.kind == .directory, previous.treeHash != nil, old.isReusable(as: entries) { var reused = previous reused.permissions = directory.permissions return reused @@ -1469,7 +1485,7 @@ actor FileCheckpointStore: CheckpointStore { } else if node.kind == .directory { let old = try children(of: node, workspaceId: workspaceId) let new = try old.map { try relocating($0, blobs: blobs, workspaceId: workspaceId) } - if old != new { + if !old.isReusable(as: new) { result = try persistDirectory(result, children: new, workspaceId: workspaceId, syncParent: true) } } @@ -2277,7 +2293,7 @@ actor FileCheckpointStore: CheckpointStore { } func compareNodes(_ old: SnapshotManifest.Node, _ new: SnapshotManifest.Node, workspaceId: UUID, capturing: Bool = false) async throws -> ChangeSet { - if old == new { return ChangeSet(changes: []) } + if old.isReusable(as: new) { return ChangeSet(changes: []) } if old.kind == .directory, new.kind == .directory { var changes: [FileChange] = (old.permissions) == (new.permissions) ? [] : [.init(path: new.path, kind: .directory, effect: .modified)] var omissions: [TextDiffOmission] = [] diff --git a/Sources/Workspace/FS/LimitedFileSystem.swift b/Sources/Workspace/FS/LimitedFileSystem.swift index e21dc7e..43a8a5f 100644 --- a/Sources/Workspace/FS/LimitedFileSystem.swift +++ b/Sources/Workspace/FS/LimitedFileSystem.swift @@ -69,11 +69,11 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti private let base: any FileSystem private let pathResolver: FileSystemPathResolver public let limits: FileSystemLimits - // Above this bound the same projection runs on an ephemeral full walk. + private let accountingCacheEntryLimit: Int #if os(iOS) || os(watchOS) - static let accountingCacheEntryLimit = 25_000 + private static let defaultAccountingCacheEntryLimit = 25_000 #else - static let accountingCacheEntryLimit = 250_000 + private static let defaultAccountingCacheEntryLimit = 250_000 #endif private var accounting = Accounting() private var cacheValid = false @@ -86,10 +86,16 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti private var mutationActive = false private var mutationWaiters: [CheckedContinuation] = [] - public init(base: any FileSystem, limits: FileSystemLimits) { + /// Creates a quota-enforcing decorator. `accountingCacheEntryLimit` caps retained + /// accounting entries, including the root. `nil` uses 250,000 entries (25,000 on + /// iOS/watchOS). Zero or negative values disable retention: every preflight walks + /// the tree. Above the cap, accounting is discarded after each operation; the + /// cap bounds retained entries, not peak memory during a walk. + public init(base: any FileSystem, limits: FileSystemLimits, accountingCacheEntryLimit: Int? = nil) { self.base = base pathResolver = FileSystemPathResolver(base: base) self.limits = limits + self.accountingCacheEntryLimit = max(0, accountingCacheEntryLimit ?? Self.defaultAccountingCacheEntryLimit) } deinit { observationTask?.cancel() } @@ -215,7 +221,7 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti } private func discardOversizedAccounting() { - if accounting.tree.count > Self.accountingCacheEntryLimit { + if accountingCacheEntryLimit == 0 || accounting.tree.count > accountingCacheEntryLimit { accounting = Accounting() cacheValid = false } @@ -457,7 +463,7 @@ public actor LimitedFileSystem: FileSystem, TraversingFileSystem, PreviewSemanti if generation != after { cacheValid = false; continue } cachedGeneration = after if rescan || !invalidations.isEmpty { continue } - return retain && accounting.tree.count <= Self.accountingCacheEntryLimit + return retain && accountingCacheEntryLimit > 0 && accounting.tree.count <= accountingCacheEntryLimit } } diff --git a/Sources/Workspace/FilesystemObservation.swift b/Sources/Workspace/FilesystemObservation.swift index 3862caa..100cbb0 100644 --- a/Sources/Workspace/FilesystemObservation.swift +++ b/Sources/Workspace/FilesystemObservation.swift @@ -173,7 +173,8 @@ struct FilesystemObservation: Sendable { let requiresContentCheck = new?.info.kind == .file && cache[path] != nil && ( verifyContents.contains(path) || (checkRacyContents.contains(path) && old.map { new?.canReuseContents($0, capturedAt: before.capturedAt, exactTimestamps: before.exactTimestamps) != true } != false)) - guard old != new || requiresContentCheck else { continue } + let reusable = if let old, let new { old.isReusable(as: new) } else { false } + guard !reusable || requiresContentCheck else { continue } // mtime/identity changes on directories alone are not workspace changes. if let old, let new, old.info.kind == .directory, new.info.kind == .directory, old.info.permissions == new.info.permissions { continue } diff --git a/Sources/Workspace/MetadataIndex.swift b/Sources/Workspace/MetadataIndex.swift index 343817d..8da4b4d 100644 --- a/Sources/Workspace/MetadataIndex.swift +++ b/Sources/Workspace/MetadataIndex.swift @@ -1,19 +1,42 @@ import Foundation +/// Metadata reuse may tolerate device renumbering; equality must remain exact. +protocol MetadataReusable: Equatable { + func isReusable(as other: Self) -> Bool +} + +extension Collection where Element: MetadataReusable { + /// Ordered children must have the same length and reusable metadata at every position. + func isReusable(as other: Self) -> Bool { + elementsEqual(other) { $0.isReusable(as: $1) } + } +} + /// An lstat index: directory timestamps are hints, never logical changes. struct MetadataIndex: Sendable { - struct Entry: Sendable, Equatable { + struct Entry: Sendable, MetadataReusable { var info: FileInfo var target: String? static func == (lhs: Self, rhs: Self) -> Bool { lhs.info.kind == rhs.info.kind && lhs.info.size == rhs.info.size && lhs.info.modificationDate == rhs.info.modificationDate - && FileIdentity.matchesForMetadataReuse(lhs.info.identity, rhs.info.identity) + && lhs.info.identity == rhs.info.identity + && lhs.info.path == rhs.info.path && lhs.info.linkCount == rhs.info.linkCount && lhs.info.permissions == rhs.info.permissions && lhs.target.map(PathName.init) == rhs.target.map(PathName.init) } + /// Observation compares logical metadata; paths are keyed separately and link counts + /// do not describe contents. Device renumbering alone does not invalidate a hint. + func isReusable(as other: Self) -> Bool { + info.kind == other.info.kind && info.size == other.info.size + && info.modificationDate == other.info.modificationDate + && FileIdentity.matchesForMetadataReuse(info.identity, other.info.identity) + && info.permissions == other.info.permissions + && target.map(PathName.init) == other.target.map(PathName.init) + } + func canReuseContents(_ old: Self, capturedAt: Date, exactTimestamps: Bool = false) -> Bool { info.kind == .file && old.info.kind == .file && old.info.modificationDate.map { exactTimestamps || $0.addingTimeInterval(2) < capturedAt } == true diff --git a/Tests/WorkspaceTests/DeviceIdentityTests.swift b/Tests/WorkspaceTests/DeviceIdentityTests.swift index a3be19e..329bfef 100644 --- a/Tests/WorkspaceTests/DeviceIdentityTests.swift +++ b/Tests/WorkspaceTests/DeviceIdentityTests.swift @@ -3,6 +3,36 @@ import Testing @testable import Workspace struct DeviceIdentityTests { + @Test + func `persisted equality distinguishes device renumbering from metadata reuse`() throws { + let original = FileCheckpointStore.SnapshotManifest.Node( + kind: .file, path: "/file", contentHash: "abc", contentSize: 3, + identity: FileIdentity(device: 1, inode: 42)) + var renumbered = original + renumbered.identity = FileIdentity(device: 2, inode: 42) + #expect(original != renumbered) + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + #expect(try encoder.encode(original) != encoder.encode(renumbered)) + #expect(original.isReusable(as: renumbered)) + #expect(renumbered.isReusable(as: original)) + #expect([original] != [renumbered]) + #expect([original].isReusable(as: [renumbered])) + #expect(![original].isReusable(as: [])) + var replaced = renumbered + replaced.identity = FileIdentity(device: 2, inode: 43) + #expect(!original.isReusable(as: replaced)) + #expect(![original].isReusable(as: [replaced])) + + let metadata = MetadataIndex.Entry(info: FileInfo( + path: "/file", kind: .file, size: 3, permissions: .defaultFile, + modificationDate: nil, identity: original.identity), target: nil) + var renumberedMetadata = metadata + renumberedMetadata.info.identity = renumbered.identity + #expect(metadata != renumberedMetadata) + #expect(metadata.isReusable(as: renumberedMetadata)) + } + @Test(arguments: ["unchanged", "size", "mtime", "racy", "verified", "replacement"]) func `device renumbering preserves capture reuse without bypassing content guards`(change: String) async throws { let root = try TestSupport.temporaryDirectory("DeviceCapture") diff --git a/Tests/WorkspaceTests/LimitedCacheTests.swift b/Tests/WorkspaceTests/LimitedCacheTests.swift index e2de16c..d1f6436 100644 --- a/Tests/WorkspaceTests/LimitedCacheTests.swift +++ b/Tests/WorkspaceTests/LimitedCacheTests.swift @@ -284,10 +284,12 @@ struct LimitedCacheTests { try await checkTotals(fs, base: memory) } - @Test - func oversizedAccountingIsDiscardedBetweenRejectedPreflights() async throws { - let base = LimitedCacheBackend(fixedEntryCount: LimitedFileSystem.accountingCacheEntryLimit) - let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100)) + @Test(arguments: [0, 3, 4, 5]) + func accountingCacheCapIncludesRootAndIsInclusive(cap: Int) async throws { + // Three files plus the root: only caps of at least four may retain this walk. + let base = LimitedCacheBackend(fixedEntryCount: 3) + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100), + accountingCacheEntryLimit: cap) try await fs.revalidate() await base.clearReads() for _ in 0..<2 { @@ -295,7 +297,30 @@ struct LimitedCacheTests { try await fs.createFile(path: "/probe", data: Data(count: 101)) } } - #expect(await base.listed.filter { $0.isRoot }.count == 2) + #expect(await base.listed.filter { $0.isRoot }.count == (cap < 4 ? 2 : 0)) + } + + @Test(arguments: [0, 2]) + func successfulMutationsRespectAccountingCacheCap(cap: Int) async throws { + let base = LimitedCacheBackend() + let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100), + accountingCacheEntryLimit: cap) + try await fs.createFile(path: "/a", data: Data(count: 10)) + await base.clearReads() + for _ in 0..<2 { + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 111, limit: 100)) { + try await fs.createFile(path: "/probe", data: Data(count: 101)) + } + } + #expect(await base.listed.filter { $0.isRoot }.count == (cap == 0 ? 2 : 0)) + + // Adding a second file takes the retained entry count from two to three. + try await fs.createFile(path: "/b", data: Data(count: 20)) + await base.clearReads() + await #expect(throws: FileSystemLimitError.totalBytes(attempted: 131, limit: 100)) { + try await fs.createFile(path: "/probe", data: Data(count: 101)) + } + #expect(await base.listed.filter { $0.isRoot }.count == 1) } @Test From 7b41186bab048b258559379de26f07129b95fb8f Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 21:17:46 -0700 Subject: [PATCH 27/34] Rewrite the README for users The README is reorganised around what an adopter needs: when to use the package, the core model, the main tasks, and a single section of guarantees and limits. Internal storage, locking and syscall detail is removed, repeated explanations are merged, every example is checked to compile and run against the public API, and benchmark detail moves to Benchmarks/README.md. --- Benchmarks/README.md | 51 +++ README.md | 792 ++++++++++++++++--------------------------- 2 files changed, 345 insertions(+), 498 deletions(-) create mode 100644 Benchmarks/README.md diff --git a/Benchmarks/README.md b/Benchmarks/README.md new file mode 100644 index 0000000..dca2708 --- /dev/null +++ b/Benchmarks/README.md @@ -0,0 +1,51 @@ +# Workspace benchmarks + +`WorkspaceBenchmarks` measures Workspace operations against generated fixtures. It builds on macOS and Linux only and doesn't run as part of `swift test`. + +## Running + +```sh +swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/before.json +swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/after.json \ + --compare Benchmarks/Results/before.json +swift run -c release WorkspaceBenchmarks --preset medium --filter transaction,limitedWrite +``` + +Scenario IDs have the form `backend/persistence/operation`, such as `local/directory/checkpointIncremental`. `--filter` takes comma-separated substrings of those IDs. `--compare` prints timing ratios and exact count differences against a saved report; compare the same preset on the same machine. `--json` rewrites the report after every scenario. Run `--help` for iteration counts, warmups and deadlines. + +| Preset | Regular files | Size | Rough time at defaults | +| --- | --- | --- | --- | +| `small` | 1,000 | about 20 MiB | 2 to 5 minutes | +| `medium` | 10,000 | about 200 MiB | 10 to 30 minutes | +| `large` | 100,000, deeply nested | about 98 MiB | well over an hour | + +The `large` preset runs only the local backend. Times depend on the filesystem and whether native watchers are available. + +## Method + +- Each scenario runs in a fresh child process, with one warmup and five measured samples by default. A child that exceeds the deadline (30 minutes by default) is killed and reported as failed. +- Fixture generation and per-sample setup are excluded from timing and counters. Write scenarios use independently prepared stores, so `write1001` always times exactly the 1,001st mutation. +- Without `--fixture-dir`, fixtures and scratch trees live in the temporary directory and are removed on completion or interrupt. `--fixture-dir PATH` generates a fixture there, or validates and reuses one already there. `--keep-fixture` preserves a generated one. +- If any scenario fails, the run exits nonzero after writing the successful results. Run outside a restrictive macOS sandbox, or the FSEvents scenarios fail. + +| Scenario | What's timed | Setup excluded | Counter blind spots | +| --- | --- | --- | --- | +| `writeText`, `apply50`, `limitedWrite` | One recorded write, 50 edits, a quota-limited write with warm accounting | No-op preparation, file reset, priming the limit accounting | Checkpoint store; Limited's metadata traversal is visible | +| `transaction`, `transactionFirst` | Begin, one draft write, commit | Preparation; for `transaction` only, an initial begin and discard and a baseline checkpoint | Draft filesystem, checkpoint store | +| `preview` | One edit preview | Preparation | Draft filesystem | +| `checkpointFirst`, `checkpointIncremental` | First capture; capture after one change | Preparation; for incremental, the baseline and the change | Checkpoint store | +| `diff`, `restore` | Checkpoint-to-current diff or restore after one change | Preparation, baseline, change | Checkpoint store | +| `readCheckpoint`, `readCheckpointWarm`, `readCurrent` | Cold and primed checkpoint reads; a current read | Baseline; the first read for the warm variant | Checkpoint store | +| `search`, `glob` | Search for an absent literal; a `**/*` glob | Preparation | Glob internals; `globMatches` is visible | +| `history1000`, `history1000Cached` | Loading 1,000 mutations in a fresh directory-backed Workspace; reading cached in-memory history | Seeding 1,000 mutations | Checkpoint store | +| `write1001` | The 1,001st recorded directory-backed write | Preparation and 1,000 mutations | Checkpoint store | +| `eventsInitial`, `eventsExternal` | Starting observation; an external write until its event arrives | Preparation; for external, the initial observation | The external write itself; recaptures are visible | + +## Reading the results + +The table reports median and p90 wall time (linear interpolation between samples), backend bytes and calls read, `stat` and list calls, entries visited, glob matches, logical store bytes and files written, and memory: setup RSS high-water, its increase during the operation, RSS held with the result alive, and that held figure for the first run. + +- Counters observe calls at the `FileSystem` boundary, not system calls. They exclude checkpoint-store I/O and a backend's internal glob traversal. Scoped walks report `walk` and `walkEntry` counters, and their content reads still count as reads. +- Store writes are estimated from directory metadata before and after the operation: appended growth or the final size of created and replaced files. Temporary and deleted files, repeated overwrites and OS write amplification don't show. +- After the first run, allocator reuse hides most of a result's memory, so the measured-sample memory figures can be near zero or slightly negative. Use the first-run column for a result's live footprint. Zero high-water growth can also mean reuse. +- The JSON report keeps every sample, including warmups, and each child's peak RSS. RSS is normalized to bytes on both platforms. diff --git a/README.md b/README.md index 351624d..7821eb3 100644 --- a/README.md +++ b/README.md @@ -1,89 +1,108 @@ # Workspace -`Workspace` is a Swift package for agent and tool runtimes that need controlled filesystem access, structured edits, revision history, and isolated transactions without shell parsing. +Workspace is a Swift package that gives agent and tool runtimes controlled access to a directory tree. It provides structured, atomic edits with a record of what changed, checkpoints you can diff and restore, isolated transactions with a three-way merge, change events, text search and an LLM tool adapter. Filesystem wrappers add root confinement, authorization rules, quotas, overlays and mounts. -The package is beta software. Its rooted filesystems, authorization rules, and limits are useful safety layers, but they are not a hardened process sandbox. +The package hasn't been released, and its API can change. -## Installation +## When to use it + +Use Workspace when a program, such as a coding agent, edits files on someone's behalf and needs to preview edits, apply them all-or-nothing, show what changed and roll back. + +It's the wrong tool when: + +- **You need to contain untrusted code.** The wrappers constrain calls made through the `FileSystem` API, not processes. Use an OS sandbox or a container. See [Safety layer, not a sandbox](#safety-layer-not-a-sandbox). +- **You need to run commands.** There's no process execution or shell. +- **You want version control.** Checkpoints are local snapshots of one tree, with no branches or remotes. +- **Every operation must be cheap on a very large tree.** Checkpoints, transaction begin and commit, and checkpoint-to-current diffs each walk the whole tree's metadata. +- **Every writer must be recorded.** Changes made outside the Workspace can appear as events but never enter history. + +## Requirements + +Swift 6.2 or later on macOS 13, iOS 16, tvOS 16, watchOS 9 or Linux, with no dependencies. Events for external changes need macOS (FSEvents) or Linux (inotify). ```swift -.dependencies: [ - .package(path: "../Workspace") +dependencies: [ + .package(path: "../Workspace"), +], +targets: [ + .target(name: "MyTool", dependencies: [ + .product(name: "Workspace", package: "Workspace"), + ]), ] ``` -The package contains one library product, `Workspace`, and has no external dependencies. - -## The Model +## Core model -The public API centers on a small set of concepts: +- **`Workspace`** is an actor that owns one tree: its current state (a `FileSystem`), a mutation history, checkpoints and an event stream. +- **`WorkspacePath`** is a normalized absolute path. String literals convert to it. +- **`Edit`** describes one mutation: write, append, create a directory, remove, copy, move, link, set permissions, or find-and-replace. +- **`ChangeSet`** lists `FileChange` values (path, kind, effect, optional `TextDiff`) and `textDiffOmissions` for files without a text diff. Previews, edit results, history, events, diffs and commits all return it. +- **`Checkpoint`** is a stored revision of the whole tree. A **`Revision`** is `.current` or `.checkpoint(id)`. +- **`Workspace.Transaction`** is an isolated draft that commits back with a three-way merge. +- **`FileSystem`** is the storage protocol behind the current state. -- `Workspace` owns current mutable state, history, checkpoints, and events. -- `Revision` addresses either the current state or a checkpoint. -- `Edit` describes a mutation. -- `ChangeSet` is the common result used by previews, edits, history, events, diffs, and transactions. -- `Workspace.Transaction` provides isolated preview, three-way commit, and discard. -- `FileSystem` is the low-level protocol for custom storage backends. +`Workspace.init` parameters, all optional: -## Create a Workspace +| Parameter | Default | Meaning | +| --- | --- | --- | +| `workspaceID` | `UUID()` | Identifies this workspace in a store. Pass the same value each time to reopen persisted history. | +| `fileSystem` | `InMemoryFileSystem()` | Holds the current state. | +| `persistence` | `.memory` | `.memory` keeps checkpoints and history in the process; `.directory(URL)` stores them on disk. | +| `recording` | `.full(maxTextBytes: 1_000_000)` | Detail kept in history: `.full` keeps text diffs up to the size limit, `.changesOnly` keeps paths and effects, `.off` keeps nothing. Returned results and events are unaffected. | +| `checkpointCapture` | `.trustMetadata` | See [Change detection](#change-detection). | +| `observation` | 32 MiB cache | See [External changes](#external-changes). | -An empty in-memory workspace needs no configuration: +## Getting started ```swift import Workspace let workspace = Workspace() +try await workspace.createDirectory("/notes") try await workspace.writeText("/notes/todo.txt", "ship it\n") let text = try await workspace.readText("/notes/todo.txt") ``` -Use a rooted local filesystem and persisted revision store when state should survive process exit: +`InMemoryFileSystem` requires parent directories to exist before a write; `LocalFileSystem` creates them. + +To work on a directory on disk and keep history across launches: ```swift -let files = try LocalFileSystem(root: projectURL) let workspace = Workspace( - workspaceID: projectID, - fileSystem: files, - persistence: .directory(historyURL), - recording: .full(maxTextBytes: 1_000_000) + workspaceID: projectID, // a UUID you store and reuse + fileSystem: try LocalFileSystem(root: projectURL), + persistence: .directory(historyURL) ) ``` -The injected filesystem remains available as `workspace.fileSystem` so one `Workspace` can be -shared as the filesystem authority across shells, tool runtimes, and other adapters. Reads through -that reference are always fine. Mutations made directly through it bypass Workspace history, -checkpoints, and events, so integrations should either route mutations through the Workspace edit -pipeline or deliberately accept that those changes are unrecorded. +`LocalFileSystem(root:)` creates the root if it's missing. A history directory holds one subdirectory per `workspaceID`. Keep it outside `projectURL`, or the store's files become part of the tree it records. -## Edits and Changes +`workspace.fileSystem` returns the filesystem you passed in. Reading through it is fine, but writes through it bypass history and the [operation gate](#atomicity-and-the-operation-gate). -Every mutation returns the same structured `ChangeSet`: +## Editing files ```swift -let preview = try await workspace.preview([ +let edits: [Edit] = [ .createDirectory("/Sources"), - .writeText("/Sources/main.swift", "print(\"hello\")\n") -]) + .writeText("/Sources/main.swift", "print(\"hello\")\n"), +] -let result = try await workspace.apply([ - .createDirectory("/Sources"), - .writeText("/Sources/main.swift", "print(\"hello\")\n") -]) +let preview = try await workspace.preview(edits) // nothing is written +let result = try await workspace.apply(edits) // atomic by default -print(result.changes.touchedPaths) +for change in result.changes.changes { + print(change.effect, change.kind, change.path) +} print(result.changes.statistics.additions) ``` -`apply` is atomic by default. `.stopOnError` and `.continueAfterError` expose partial failures explicitly. -Atomic recovery does not track hard-link aliases outside captured roots or roll back permission changes to symlink entries. +`preview` runs the edits on a copy-on-write layer and returns the ChangeSet `apply` would produce. It can't predict concurrent writes or permission and quota failures. -Edit previews use an isolated copy-on-write layer and the same affected-root journal as `apply`. -Their ChangeSets describe the edited paths, including link and permission changes. Preview does not -simulate external races or guarantee that later writes pass filesystem permissions or quota checks. +`apply(_:policy:)` is `.atomic` by default: if any edit fails, the affected paths are restored and the error is rethrown. `.stopOnError` keeps earlier successes and stops at the first failure; `.continueAfterError` attempts every edit. Both report failures in `result.failures` and record what succeeded. -Direct conveniences—`writeText`, `writeData`, `appendText`, `createDirectory`, `remove`, `copy`, `move`, link creation, and permission changes—use the same edit pipeline. +`writeText`, `writeData(_:to:)`, `writeJSON`, `appendText`, `appendData`, `createDirectory`, `remove`, `copy`, `move`, `createSymbolicLink`, `createHardLink` and `setPermissions` each apply one edit and return its ChangeSet. -Text replacement is an edit and shares its file selection and pattern engine with search: +Find-and-replace is an edit like any other: ```swift let swiftFiles = FileSelection( @@ -97,161 +116,82 @@ try await workspace.apply([ ]) ``` +Patterns are `.literal(_:caseSensitive:)` or `.regularExpression(_:)` (an `NSRegularExpression`; the replacement accepts `$1` templates). Every selected file must be UTF-8 text without NUL bytes, or the edit fails. + +Selection patterns resolve against `root`; includes are combined and excludes removed. Globs support `*`, `?`, `[...]`, `[!...]` and `**`, and `*` matches names starting with a dot. + +`readText`, `readData` (with an optional byte range), `readJSON`, `exists`, `info`, `list`, `glob` and `tree` all accept a `Revision`, so they read checkpoints too. + ## Search ```swift -let result = try await workspace.search( - SearchRequest( - pattern: .regularExpression(#"\bTODO\b"#), - files: swiftFiles, - contextLines: 2 - ) +let request = SearchRequest( + pattern: .regularExpression(#"\bTODO\b"#), + files: swiftFiles, + contextLines: 2 ) +let result = try await workspace.search(request) for match in result.matches { print("\(match.path):\(match.lineNumber): \(match.line)") } ``` -Search is deterministic, line-oriented, does not follow symlinks, and reports binary, oversized, or truncated work as structured result data. - -## Checkpoints, Revisions, and Diffs +Search reads regular files in the current state, in path order, without following symbolic links. Binary and oversized files are listed in `skippedFiles`. Default limits are 1,000,000-byte files, 10,000 files and 1,000 matches; `maxOutputBytes` can also cap the encoded result. `truncation` names the limit that stopped a result. To page, pass `nextMatchOffset` back as `matchOffset`. Each page rescans every selected file. -Checkpoints are persisted revisions; there is no second public snapshot representation. +## Checkpoints, diffs and restore ```swift +try await workspace.writeText("/README.md", "Original\n") let before = try await workspace.createCheckpoint(label: "before") try await workspace.writeText("/README.md", "Updated\n") -let after = try await workspace.createCheckpoint(label: "after") -let oldText = try await workspace.readText( - "/README.md", - at: .checkpoint(before.id) -) +let oldText = try await workspace.readText("/README.md", at: .checkpoint(before.id)) +let changes = try await workspace.diff(from: .checkpoint(before.id), to: .current) +print(changes.unifiedPatch()) -let changes = try await workspace.diff( - from: .checkpoint(before.id), - to: .checkpoint(after.id) -) +let restored = try await workspace.restore(to: before) ``` -Checkpoint reads use cached in-memory indexes or persisted manifest metadata, resolve directory -symlinks inside the workspace namespace (at most 64 hops), and read selected byte ranges without -materializing whole blobs. `TextDiff` includes total line counts, addition/deletion statistics, -hunks, and UTF-16 intra-line ranges. Empty and identical inputs and unchanged prefixes/suffixes -avoid the general diff search. A Myers search preserves sparse edits; if its work or edit-distance -limit is reached, the unmatched interior uses a coarse replacement hunk. Its statistics describe -the emitted additions/deletions and may exceed a minimal edit script. The line search and each -intra-line pair have independent limits of 1,000 edits or 1,000,000 visited search steps. +A checkpoint records file contents, directories, symbolic links and permissions, storing identical contents once. Its `origin` is `.manual`, `.rollback(from:)` or `.transaction(base:)`, and its `summary` describes changes from its parent. + +`restore(to:)` rewrites only the entries that differ, then records the result as a checkpoint with origin `.rollback(from:)`. Later checkpoints are kept. -Use `Workspace.Archive` only when a fully materialized, portable subtree is actually required: +`diff(from:to:options:)` compares any two revisions. Files larger than `DiffOptions.maxTextBytes` (default 1,000,000), binary files and metadata-only changes get no text diff and are listed in `textDiffOmissions`. `ChangeSet.unifiedPatch()` renders a git-style patch. The diff search gives up after 1,000 edits or 1,000,000 steps per file and emits one replacement hunk for the unmatched middle, so statistics can exceed a minimal diff. + +To copy a subtree into another workspace, use an archive. Restoring one is recorded in history but creates no checkpoint. ```swift let archive = try await workspace.archive(at: "/Sources") try await anotherWorkspace.restore(archive, at: "/Imported") ``` -## Storage Lifecycle - -Inspect checkpoint-store usage and preview retention before deleting anything: +### Compaction ```swift -let usage = try await workspace.storageStatistics() - -let preview = try await workspace.compact( - retaining: .latest( - 50, - preservingLabeled: true, - preserving: [releaseCheckpoint.id] - ), +let plan = try await workspace.compact( + retaining: .latest(50, preservingLabeled: true, preserving: [release.id]), dryRun: true ) +print(plan.removedCheckpointCount, plan.reclaimedBytes) -print(preview.removedCheckpointCount) -print(preview.reclaimedBytes) -``` - -Apply the same policy by omitting `dryRun`: - -```swift let report = try await workspace.compact(retaining: .latest(50)) ``` -Compaction rebases retained checkpoints to their nearest retained ancestor and recomputes their summaries. Retained rollback and transaction checkpoints also retain an existing checkpoint they reference as provenance. `.all` keeps every checkpoint while sweeping orphaned manifests and blobs. - -Directory-backed stores serialize checkpoint commits, revision reads, and compaction with a persistent lifecycle lock. Mutation history is independent and is not pruned by checkpoint retention. - -Directory stores use the marker `{"format":"workspace-history","version":1}` in `format.json`. -An existing store directory must have that marker; a missing, malformed, or unsupported marker is an error instructing the -caller to delete the history directory. Store creation is serialized across instances and processes. - -A small snapshot root points to SHA-256-addressed directory -objects in `trees/`. Each object contains byte-sorted immediate child names (never absolute paths), with child directories -referenced by hash. An incremental capture writes only changed directories and their ancestors; -unchanged objects remain shared. File reads decode only directories on the requested path. -Decoded trees and the write catalog share a **32 MiB estimated decoded-storage budget** per -store (24 MiB for trees, 8 MiB for the catalog). Both refresh recency on hits and normally use -O(1) LRU eviction. Capture-time tree inserts evict the most recent entry instead, so repeated -whole-tree scans retain a resident set when the trees exceed the budget. Estimates include entry, -string-buffer and bookkeeping costs; this is not a process-RSS cap. -File metadata and directory listings also use path-only lookup. Full revision indexes are still materialized -when a whole-tree operation requests them. Metadata capture still walks the filesystem. - -Blobs retain their SHA-256 identities. Small captures use loose `blobs/` files; after the -first 32 new unique blobs in a capture, remaining blobs go into immutable `packs/.pack` -files with adjacent JSON hash/range indexes. Packs target 16 MiB and use a 1 MiB write buffer; -a single larger file may exceed the target. Tree file entries carry their pack/range, so reads -need no global pack-index decode. Full snapshot materialization maps at most four packs at a -time and copies only the required file ranges; those mappings do not escape the locked load. -The bounded write-side pack catalog is cached for one workspace. A constant-size `packs.generation` -token invalidates other writers' catalogs before compaction removes packs, without depending on -directory mtimes or enumerating historical pack files on warm writes. Hashes captured with in-memory -checkpoints are reused by revision indexes and unchanged files in subsequent checkpoints. - -Publication orders durable blob/pack writes and directory objects before the snapshot root, -and the snapshot root before checkpoint metadata. Files are synchronized before atomic rename, -and containing directories are synchronized before advancing publication. Tree-directory syncs -are batched at capture completion; compaction synchronizes each relocated tree before replacing -roots. On Darwin, snapshot-root and checkpoint-metadata publication additionally use `F_FULLFSYNC` -after syncing the containing directory, flushing the drive cache including preceding writes. -A successfully published checkpoint therefore survives process crashes and power loss on storage -that honors these synchronization requests. If `F_FULLFSYNC` fails other than interruption, -publication retains the completed file and directory `fsync`; power-loss protection is best-effort -on such volumes. Other POSIX platforms use file and directory `fsync`. -Failed captures can leave unreachable objects, but publish no checkpoint referencing incomplete content. Compaction -sweeps unreachable objects, rewrites partially live packs, durably replaces affected snapshot roots, -and only then removes old objects. Interrupted compaction can be resumed; dry-run blob counts -and byte totals include reclaimable pack entries and incomplete orphan packs. - -All writers preserve the mutation sequence high-water mark and retraction epoch in -`mutations.sequence.json`. Retractions and failed -append reservations can leave sequence gaps. History refreshes invalidate cached records when -another instance retracts or prunes mutations. - -Directory-backed checkpoint capture also reuses existing CAS blobs when a file's size and available -modification timestamp are unchanged and the timestamp is more than two seconds older than the -previous capture's start. Entries within that timestamp-granularity window are rehashed. It still -walks directory metadata for correctness. Filesystems without modification timestamps and -materialized snapshots without capture hints require content reads. - -For tools that preserve older timestamps while changing content, pass -`checkpointCapture: .verifyContents` to `Workspace` to read and hash every file at each checkpoint. -The default `.trustMetadata` policy compares the local wall clock with filesystem modification -timestamps. Clock skew can make a recent change appear safely old; use `.verifyContents` for -network filesystems whose clock may differ. This policy affects only `.directory` persistence. +`.latest` keeps the newest checkpoints, labeled ones and any IDs you list. It also keeps the checkpoint each kept rollback or transaction checkpoint came from, and any checkpoint that an open transaction or another Workspace instance in the process depends on. `.all` keeps every checkpoint and removes only unreferenced data. Survivors are rebased onto their nearest kept ancestor. `storageStatistics()` reports usage. Nothing prunes mutation history, so use `recording` to bound it. ## Transactions -Begin an isolated transaction: - ```swift let transaction = try await workspace.beginTransaction(label: "agent draft") try await transaction.writeText("/README.md", "Draft\n") -let preview = try await transaction.preview() -let commit = try await transaction.commit(strategy: .threeWay) +let draft = try await transaction.preview() // base-to-draft ChangeSet +let commit = try await transaction.commit() // .threeWay by default -if !commit.applied { +if commit.applied { + print("committed as checkpoint", commit.checkpoint?.id as Any) +} else { for conflict in commit.conflicts { print(conflict.path, conflict.kind) } @@ -259,203 +199,87 @@ if !commit.applied { } ``` -Transaction commits follow these rules: +A transaction starts from a frozen copy of the tree and edits a private draft, with the same read, search and edit methods as `Workspace`. The parent stays editable. A successful commit writes the merge to the parent, records it in history and as a checkpoint with origin `.transaction(base:)`, and emits an event. `commit.checkpoint` is `nil` when there was nothing to apply. + +### Merge rules + +The merge compares nodes path by path. A node is a path's kind, permissions and content (file bytes or symlink target), not a directory's children. -- Three-way merging accepts a change made by only one side, coalesces identical changes, and reports divergent changes to the same node as conflicts. -- A directory removed or replaced on one side while the other creates or modifies something beneath it is a `.structural` conflict at the outermost removed or replaced directory. Descendant deletions converge with ancestor removal. Directory mode changes and child edits merge. -- Strict mode rejects any parent change, including root-directory metadata. -- Moves are delete-plus-create operations. Moving the same source to different destinations keeps both copies. -- During `commit()`, edits and `discard()` throw `TransactionError.committing`; read-only draft calls remain available. Another commit also throws `.committing`. -- A rollback failure permanently closes the transaction. Failures that leave the parent unchanged allow retry. +| Parent since begin | Transaction | Outcome | +| --- | --- | --- | +| Unchanged | Changed | Transaction's version | +| Changed | Unchanged | Parent's version | +| Same change | Same change | Converges | +| Modified | Modified differently | `bothModified` | +| Created | Created differently | `bothCreated` | +| Deleted | Modified, or the reverse | `modifiedAndDeleted` | +| Removed or replaced a directory | Changed something inside it, or the reverse | `structural`, once, at the outermost such directory | + +So a permission change on a directory merges with edits inside it, and deleting files under a directory the other side removed converges. A move is a delete plus a create: if both sides move the same file to different paths, both copies survive. Node conflicts carry `parentDiff` and `transactionDiff` when both sides are text. + +`commit(strategy: .strict)` instead rejects the commit if anything in the parent changed, including root metadata, reporting each changed path as `parentChanged`. -Manual conflicted transactions remain active. The scoped convenience commits on success and discards on failure or conflict: +### Lifecycle + +A conflicted commit applies nothing and leaves the transaction active, so you can edit the draft and commit again, or discard it. During a commit, edits, `discard()` and another `commit` throw `TransactionError.committing`; reads keep working. After a successful commit or a discard, calls throw `.inactive`. If a commit fails and the parent is restored, the transaction stays active; if the restore also fails, it throws `WorkspaceRollbackError` and the transaction closes. + +The scoped form commits when the body returns and discards when it throws or conflicts. Conflicts throw `TransactionError.conflicts`. ```swift -let result = try await workspace.transaction { transaction in +let result = try await workspace.transaction(label: "agent run") { transaction in try await transaction.writeText("/result.txt", "done\n") return "agent result" } +print(result.value, result.commit.appliedChanges.touchedPaths) ``` -Transactions use a copy-on-write draft over a frozen base. The first transaction on an instance -reads the tree, including after reopening a Workspace. Memory persistence retains the initial bytes. -A cold directory-backed base retains up to 256 MiB of file contents in memory; files that do not fit -use private temporary blobs until a checkpoint supplies immutable content. Live transaction handles -own the cold base. Discarding or releasing the last handle releases its bytes; successful commit -releases the completed handle's base and re-seeds the parent from the checkpoint. Other active handles -retain their frozen contents. A later begin without a live cold base or checkpoint base reads anew. -Beginning or discarding a transaction adds no public checkpoint. Subsequent begins and commits walk -metadata and read changed or unverifiable files; previews and merges materialize affected paths. -Metadata capture uses bulk traversal when available and per-path enumeration otherwise, including -scoped subtree refreshes. Mounted traversal preserves virtual paths and shadowing; authorized -traversal retains per-entry permission checks. -Snapshot capture likewise stats each entry on its per-path route. InMemoryFileSystem has a native -actor-owned traversal; after reentrant mutations it resolves pending paths and reads again. -Checkpoint persistence still requires the store's metadata and manifest work. Checkpoints prime a -transaction base only after that instance has begun a transaction. Directory commits replace the base -with the new revision index, without retaining changed-file contents or revision read caches. - -With `checkpointCapture: .trustMetadata`, transactions reuse contents when size and mtime match -and the previous mtime is more than two seconds older than the start of the previous metadata -capture. Identity must also match when both entries provide it. Missing mtimes and recent timestamps -require reads; InMemoryFileSystem's exact timestamps skip this racy window. Writes that preserve -safely old metadata can evade the check. Use `.verifyContents` to read and compare every file during -transaction begin and commit, checkpoint-to-current diff and restore, and to read and hash every -file during directory checkpoint capture. - -Checkpoint-to-current diffs and `restore(to:)` use one bulk metadata walk and compare against the -checkpoint's persisted capture hints and revision hashes. They read only suspect files (changed -metadata, missing hints, or timestamps inside the same two-second window). Identity is compared -when both captures provide it; exact-timestamp backends skip the window. Revisions without -capture hints conservatively require content reads. Memory persistence retains hints for only the -four most recently captured or restored revisions; older revisions also fall back to content reads. -As with transactions, writes that preserve -safely old metadata can evade `.trustMetadata`; use `.verifyContents` when those writes must be -detected and restored. - -Restore applies and recaptures only differing entries and necessary ancestor permissions. Its -failure recovery and recorded changes use the same sparse scope. The new checkpoint reuses existing -blobs and directory trees through the store's normal locked, durable publication path. Archive -restore continues to capture its selected subtree in full. - -Transactions are logically atomic within cooperative workspace use. They are not crash-safe against -an external process racing the commit. In-process `Workspace.compact` protects revisions referenced -by transaction handles and other Workspace instances. This instance's reusable base does not override -retention: it survives dry runs and real compaction that retains its checkpoint, and is discarded -when that checkpoint is removed. Coordinate compaction across processes: the store protocol has no -cross-process revision lease. - -## Events and History +Begin and commit walk the parent's metadata, and the first transaction on a Workspace instance reads every file to build its base: + +- With `.memory` persistence, the base stays in memory and later begins read only files whose metadata changed. +- With `.directory` persistence, up to 256 MiB stays in memory and the rest goes to temporary files. After a commit or checkpoint, later transactions start from that checkpoint. Otherwise, a begin after every handle has been discarded reads every file again. + +## Events and history ```swift -let events = await workspace.events() -Task { +let events = await workspace.events(.init(path: "/Sources")) +let watcher = Task { for await event in events { - print(event) + switch event { + case let .changes(changes): + print("changed:", changes.touchedPaths) + case let .checkpoint(checkpoint): + print("checkpoint:", checkpoint.id) + } } } let mutations = try await workspace.history() ``` -Change events, returned edit results, and mutation history all carry the same `ChangeSet` representation. Checkpoint events share the same stream. - -`LocalFileSystem` also pushes out-of-band filesystem invalidations through FSEvents on macOS and -inotify on Linux. `events(_:)`, with an optional path filter, turns those -invalidations into structured `ChangeSet` values. External changes remain unrecorded in mutation -history, matching the direct-filesystem contract above. Directory-backed checkpoint stores use the -same push model for cross-instance checkpoint events; there is no periodic polling loop. - -Filesystem observation keeps a metadata index and a bounded least-recently-used content cache. -`observation: .init(contentCacheBytes: 32 * 1024 * 1024)` configures the cache on `Workspace`; 32 MiB is -the default, zero disables it, and negative budgets are treated as zero. Initial observation reads -at most that budget, excluding files above the text-diff size limit. A change whose old or new bytes -are uncached has no text diff and reports `.observationCacheMiss` in `textDiffOmissions`. -`.sizeLimitExceeded` identifies files above the text-diff limit. Failed content reads report the -metadata change with `.contentReadFailed`, without blocking other changes. - -Full and recursive metadata rescans verify cached contents within the same two-second racy window -used by transactions, and whenever mtime is unavailable. Exact-timestamp backends skip that window. -Explicit path signals compare cached regular-file contents even when metadata matches. Rescans can -miss writes that preserve safely old metadata. Without cached contents, observation relies on -metadata and can report an identical rewrite as changed or miss a rewrite that preserves metadata. -Observation is not an atomic snapshot of external activity: a mutation between a Workspace write -and its metadata reconciliation can be absorbed into that write, particularly for uncached files -or structural changes. Coordinate concurrent writers when every intermediate change must be recorded. - -`FileSystemChangeSource.changeSignals()` emits `.paths([WorkspacePath: Scope])` in the source's -filesystem namespace. `.entry` checks one entry's contents and metadata; `.shallow` also checks a -directory's immediate listing without traversing existing child directories; `.recursive` checks -the whole subtree. Added, removed or replaced directories include their current or former descendants. -Sources must name deleted or replaced ancestors, use recursive scopes for directory renames, and emit -`.rescan` when coverage is uncertain. Coalescing keeps the strongest scope at each path without letting -a shallow ancestor suppress descendant signals. The default maps `changes()` Void signals to `.rescan`; -wrappers should forward richer signals when available. FSEvents and inotify provide scoped signals, -coalesced over 40 ms, with full metadata rescans for dropped or unknown events. Native watchers exclude -LocalFileSystem's atomic-replace temporary names in this process's private random namespace; similarly -named user files and temporary files from other processes remain visible. -Metadata failures retain their invalidations and retry with exponential backoff for at most three -attempts. After that, observation waits for a new native signal before resuming the pending work. - -## Path Identity - -A `WorkspacePath` is a normalized absolute path whose identity is the exact UTF-8 -bytes of each component. Equality, hashing, ordering, containment, permission -rules, and quota accounting do not fold case or Unicode normalization. For example, -`/café` (NFC) and `/cafe\u{301}` (NFD) are different paths. This deliberately changes -previous releases' use of Swift canonical-equivalence equality. Names and symlink -targets must be valid UTF-8; undecodable host entries are rejected, not repaired -into a different name. In-memory storage -and byte-exact host filesystems can hold both names. - -`LocalFileSystem` requires the requested spelling to match the stored directory -entry. On APFS, including case-sensitive APFS, case or normalization aliases are -rejected: lookups and reads report `ENOENT`; creates, writes, and rename/copy -destinations report `EEXIST` when an equivalent entry exists. Renaming an entry -to an equivalent spelling is rejected too; move through a distinct temporary -name if a spelling change is needed. Case-only renames remain possible on -case-sensitive filesystems. Removing a missing/aliased spelling remains -a no-op. Symlink targets must use the stored spelling, and `resolveRealPath` -returns that spelling. Directory listings and bulk traversal use names -emitted by `readdir`, so they already carry the stored identity. - -On Darwin, volume policy is cached by device. APFS and HFS+ use -`getattrlistat(ATTR_CMN_NAME, FSOPT_NOFOLLOW)` to verify stored names. For APFS, -an `F_GETPATH` check on the held parent **directory** can verify its components -together, against the root descriptor's own path plus the resolved physical -components; the final name still uses `getattrlistat` so hard links remain valid. -Unavailable, overlong, or mismatched directory paths fall back to per-component -checks, so relocating the pinned root or an ancestor remains supported. -No final file descriptor is checked with `F_GETPATH`. - -HFS+ and macOS's exFAT driver store Unicode names in decomposed form (NFD). -Creating a precomposed (NFC) name that decomposes there fails with `EILSEQ` and -the new entry is removed; use the NFD spelling reported by directory listings. -New entries are checked before use on rewriting volumes. A rewritten rename is -rolled back to preserve its source. Other Darwin -filesystems (including exFAT/FSKit, network and unknown types) verify spelling by -an exact byte match in the parent directory's `readdir` listing instead of trusting -name-cache-sensitive attribute APIs. Their new entries are also checked for name -rewriting. Listings are never cached, including on filesystems with coarse mtimes. - -On Linux, ext4/f2fs/bcachefs/overlayfs/tmpfs directory flags are inspected with -`FS_IOC_GETFLAGS`. Directories with `FS_CASEFOLD_FL` use listing verification; -others skip spelling verification. Flags are cached per `(device, inode)` in a -bounded 4,096-entry cache under the existing exclusion of external host namespace -and flag changes. A full cache is cleared before adding a new entry. Workspace -creation/removal invalidates directory entries to prevent stale flags on inode -reuse. Changing casefold requires ownership and an empty directory and is not -exposed by the workspace API. Inspecting flags requires read permission; -unavailable inspection falls back to listing verification, which also requires -read access. While its policy remains cached, a byte-exact directory can still -be traversed with search-only access. - -XFS volume geometry (`XFS_IOC_FSGEOMETRY`) is queried once per device: ascii-ci -volumes use listing verification and ordinary XFS skips it. Known byte-exact btrfs -and ramfs also skip it. Unknown filesystems, and unavailable sensitivity queries, -use listing verification rather than being refused. An existing kernel lookup -without an exact listing match is `ENOENT` for reads or `EEXIST` for creation; -missing creations use exclusive creation. Errors do not disclose the stored name. - -These checks do not make metadata checks and later mutations atomic against an -external process renaming entries; the existing concurrent-host mutation -limitations still apply. Directory names emitted by `readdir` are already verified -spelling and are not looked up a second time solely to check their spelling. - -## Filesystems and Safety Layers - -Built-in filesystems use consistent names and constructor-driven configuration: - -- `InMemoryFileSystem` -- `LocalFileSystem` -- `OverlayFileSystem` -- `MountedFileSystem` -- `SandboxFileSystem` -- `SecurityScopedFileSystem` -- `AuthorizedFileSystem` -- `LimitedFileSystem` - -Compose wrappers explicitly: +`.changes` events cover edits, restores, archive restores and commits made through this Workspace, plus external changes the filesystem reports. Events don't say which. `.checkpoint` events cover this instance's checkpoints and, with `.directory` persistence, those of other instances sharing the store. Cancel the consuming task to end a stream. + +`history()` returns `Mutation` records for edits, rollbacks, transactions and archive restores from every Workspace sharing the store. It waits for this instance's in-flight mutations. External changes never enter history. + +### External changes + +`LocalFileSystem` reports external writes through FSEvents on macOS and inotify on Linux, and the wrappers over it forward those reports. `AuthorizedFileSystem` forwards only paths the authorizer lets you `stat`. `InMemoryFileSystem` and `OverlayFileSystem` report nothing. Custom backends can adopt `FileSystemChangeSource`. + +On a filesystem that reports changes, the first `events` call walks the tree's metadata and fills a content cache. A text diff for an external change needs the file's previous contents from that cache. It's least-recently-used and sized by `ObservationPolicy(contentCacheBytes:)`: 32 MiB by default, 0 to disable. Files over 1,000,000 bytes are never cached. When the diff can't be computed, the change is still reported, with an omission reason: `.observationCacheMiss`, `.sizeLimitExceeded` or `.contentReadFailed`. + +Without cached contents, observation relies on metadata: an identical rewrite can show as modified, and a rewrite that keeps size and an old modification time can be missed. An external write that races a Workspace edit can be folded into that edit's event, so coordinate writers if every intermediate change matters. + +## Filesystems and safety wrappers + +| Type | Role | +| --- | --- | +| `InMemoryFileSystem()` | Default backend for tests, previews and scratch trees. | +| `LocalFileSystem(root:)` | A host directory, confined to its root. Reports external changes on macOS and Linux. | +| `SandboxFileSystem(root:)` | A local filesystem at `.documents`, `.caches`, `.temporary`, `.appGroup(_:)` or `.url(_:)`. | +| `SecurityScopedFileSystem(url:mode:)` | A security-scoped URL, read-only or read-write, with bookmarks kept in a `BookmarkStore`. | +| `OverlayFileSystem(over:)` | Copy-on-write: reads pass through, writes stay in memory until `discardChanges()`. | +| `MountedFileSystem(base:mounts:)` | Several filesystems as one tree. A move between mounts copies, then removes. | +| `AuthorizedFileSystem(base:authorizer:)` | Checks every operation against a policy. | +| `LimitedFileSystem(base:limits:)` | Rejects mutations that would exceed byte, entry or per-write limits. | ```swift let local = try LocalFileSystem(root: projectURL) @@ -468,130 +292,147 @@ let limited = LimitedFileSystem( ) ) -let rules = RuleBasedPermissionAuthorizer( - rules: [ - PermissionRule( - operations: [.readFile, .listDirectory, .stat], - pathPrefix: "/Sources", - effect: .allow - ) - ] +let reads: Set = [ + .stat, .exists, .listDirectory, .readFile, .readSymlink, .resolveRealPath, .glob, +] +let writes: Set = [ + .writeFile, .createDirectory, .remove, .move, .copy, + .createSymlink, .createHardLink, .setPermissions, +] +let authorizer = RuleBasedPermissionAuthorizer(rules: [ + PermissionRule(operations: reads, pathPrefix: "/", effect: .allow), + PermissionRule(operations: writes, pathPrefix: "/Sources", effect: .allow), +]) + +let workspace = Workspace( + fileSystem: AuthorizedFileSystem(base: limited, authorizer: authorizer) ) +``` + +Put `LimitedFileSystem` directly over the storage backend and `AuthorizedFileSystem` outside it. Over an `AuthorizedFileSystem` base, `LimitedFileSystem` can't reuse its accounting and walks the whole tree before every mutation. + +### Authorization + +Each call becomes a `PermissionRequest`. A path through a symbolic link is checked a second time at its resolved target, and both checks must pass. Denials throw `WorkspaceError.accessDenied`. -let authorized = AuthorizedFileSystem(base: limited, authorizer: rules) -let workspace = Workspace(fileSystem: authorized) +`RuleBasedPermissionAuthorizer` uses the first rule that names the operation and matches the paths, falling back to `defaultEffect`, which denies by default. Prefixes match whole components, so `/Src` doesn't match `/Sources`. An allow rule must cover every path in a request, including both ends of a copy or move; a deny rule applies if any path matches. Workspace edits also make `stat`, `exists`, `resolveRealPath`, `listDirectory` and `glob` calls, so allow those wherever edits are allowed. + +For interactive approval, `PermissionAuthorizer` wraps a handler. It can return `.allow` for one request, `.allowForSession`, `.allowFor(_:)` a `Duration`, or `.deny(message:)`. Both authorizers keep a bounded audit log. + +```swift +let approvals = PermissionAuthorizer { request in + switch request.operation { + case .stat, .exists, .listDirectory, .readFile, .glob, .resolveRealPath: + return .allowForSession + default: + return .deny(message: "ask the user before \(request.operation)") + } +} ``` -`LimitedFileSystem` serializes its own preflight and mutation with a FIFO gate. Quotas -count logical regular-file bytes per visible path (including hard-link aliases), and -all non-root entries, including dangling symlinks. Appends charge aliases by identity -and link count; unknown aliases are charged conservatively. Already over-quota trees -may shrink or preserve usage, but cannot increase an exceeded quota. Accounting reads -metadata only and resolves symlinks in the backend's namespace, including mounts. - -When the base declares complete change-signal coverage, Limited caches totals, compact -per-path accounting metadata (kind, size, identity and link count), children, -and alias indexes. Successful mutations refresh their affected metadata under the gate; -a failed or uncertain mutation discards the cache. File and directory identities are -needed to rule out unrecognized aliases through compositions (even for single-link -files); missing identities make the mutation uncertain and force a fresh walk on the -next preflight. External `.entry`, `.shallow`, and -`.recursive` signals invalidate those scopes for lazy reconciliation on the next -preflight. Removed/replaced directories invalidate their former descendants; hard-link -aliases are refreshed together. `.rescan` discards all cached accounting. A failed or -ended subscription falls back to a full metadata walk on every preflight, unless an -exact mutation generation is available. Coverage is checked at every preflight; Mounted -declares complete coverage only when its base and every mount do, including later mounts. -Incomplete or undeclared coverage uses full walks (or the exact generation capability). -Each Limited over Local registers its own recursive inotify watcher on Linux; large trees -can exhaust the watch limit (ENOSPC), at which point accounting safely falls back to walks. -`LimitedFileSystem(base:limits:accountingCacheEntryLimit:)` caps retained accounting entries, -including the root. Pass `nil` (the default) for 250,000 entries (25,000 on iOS/watchOS), -or an explicit entry count to tune memory retention. `0` (or a negative value) disables -retention, so every preflight walks the tree. Above the cap, accounting is discarded -after the operation. This bounds retained entries, not peak memory during a walk. - -**Consistency:** a quota decision incorporates signals processed by this Limited actor -before its projection finishes. Signals buffered in the backend or stream, or queued -for actor processing, may arrive after a decision, so external writes can temporarily make accounting stale and -can exceed quotas. `try await limited.revalidate()` forces a fresh metadata walk under -the gate without waiting for signals. With external writers quiescent, it closes the -notification-delivery gap. Neither revalidation nor the gate makes preflight atomic -with writers using another wrapper or the base directly. The package-level `changeSignalsAreComplete` -capability is an explicit guarantee of coverage for all visible children; sources that -only provide a partial stream cannot enable notification-based accounting reuse. - -Without complete change signals, each preflight walks the whole tree. The package's optional -exact mutation-generation capability lets InMemoryFileSystem reuse accounting when -unchanged; direct mutations, reset, and partial failures change its generation and -force a new walk. Generation changes during a scan or this wrapper's mutation prevent -reuse of that potentially inconsistent snapshot. -Accounting retains strict bulk-walk semantics: a listed entry vanishing before metadata -lookup fails the preflight and invalidates partial accounting; the next preflight scans anew. - -Authorization supports one-shot, duration-limited, and session approvals plus bounded audit records. Prefix rules use path-component boundaries and require both paths to match for copy and move. - -Local scans record symlinks without descending into them and retain per-entry authorization -checks when using an authorized filesystem. Scans are live reads, not atomic snapshots: -concurrent changes can affect the result or cause an error. Snapshot capture fails when a -listed child disappears before it can be inspected or read. Directory-backed checkpoints omit -listed children that vanish before their metadata lookup (ENOENT/ENOTDIR); failures after metadata, -including content reads, still abort the checkpoint. Scoped no-follow opens reject -substituted symlinks even when they point inside the root. Deep scans bound open directory -handles by falling back to ordinary per-path operations; no handles are cached between calls. -As with other local operations, host processes can relocate open directories or mutate -hard-linked data. - -## LLM Tool Adapters - -`WorkspaceToolAdapter` exposes dependency-free function definitions for read, glob, search, apply, -diff, and checkpoint operations. Each definition contains an ordinary JSON Schema object, so an -integration can translate it into its model provider's tool format without adding a provider SDK to -Workspace: +### Limits + +- `maxTotalBytes` caps the sum of regular-file sizes, counted per path, so each hard link counts. +- `maxEntryCount` caps entries other than the root, including dangling symbolic links. +- `maxWriteBytes` caps the data in one write or append. + +Violations throw `FileSystemLimitError`. A tree already over a limit can shrink but not grow. Accounting reads metadata only. Mutations through one `LimitedFileSystem` run one at a time. + +**Consistency rule.** A quota decision includes every change notification the `LimitedFileSystem` has processed before it finishes projecting the mutation. Notifications still in flight can arrive later, so external writes can leave accounting briefly stale and push usage past a limit. `try await limited.revalidate()` re-walks the metadata without waiting for notifications, which closes that gap while external writers are quiet. Neither makes the check atomic with writes that bypass this instance. + +Accounting is reused between mutations over `InMemoryFileSystem`, and over `LocalFileSystem` on macOS and Linux, including through `SandboxFileSystem`, `SecurityScopedFileSystem` or a `MountedFileSystem` whose parts all qualify. Otherwise each mutation walks the tree first. On Linux each instance holds its own recursive inotify watch and falls back to walks if the system's watch limit runs out. + +Retained accounting is capped by `accountingCacheEntryLimit` on `LimitedFileSystem.init`. The default (`nil`) keeps up to 250,000 entries, or 25,000 on iOS and watchOS; `0` keeps nothing between mutations, so every mutation walks the tree. Larger trees still work, they just walk the tree before each mutation. + +## LLM tool adapter ```swift let adapter = WorkspaceToolAdapter(workspace: workspace) -let definitions = WorkspaceToolAdapter.definitions +let definitions = WorkspaceToolAdapter.definitions // name, description, inputSchema -let resultJSON = try await adapter.call( - name: "workspace_diff", - arguments: Data(#"{"from_checkpoint_id":"...","unified_patch":true}"#.utf8) +let output: Data = try await adapter.call( + name: "workspace_read", + arguments: Data(#"{"path":"/README.md"}"#.utf8) ) ``` -All mutating adapter calls use the Workspace edit/checkpoint APIs, so they retain the same history, -event, transaction, authorization, and limit behavior as direct callers. The diff result carries the -structured `ChangeSet` and can optionally include unified-patch text for prompt or CLI interop. - -Tool arguments use snake_case and results use camelCase. Output budgets for glob, diff, and apply -count **encoded JSON bytes**, including escaping and metadata: `max_bytes` defaults to 65,536, -has a minimum of 1,024, and clamps to 262,144. Diff and apply also accept `max_file_bytes` for each -file's encoded text diff plus optional patch; it defaults to 8,192, clamps to 65,536, and accepts 0 -to omit all text detail. Returned patch sections are complete; text that cannot fit is omitted -with `textDiffOmissions.reason = "outputBudgetExceeded"`. +Each definition's `inputSchema` is a `JSONValue` holding a plain JSON Schema object, ready to translate into a provider's tool format. Arguments are snake_case JSON; results are camelCase JSON. Mutating calls go through the Workspace, so history, events, authorization and limits apply. -| Tool | Additional arguments | Result | +| Tool | Purpose | Paging and size arguments | | --- | --- | --- | -| `workspace_glob` | `match_offset` (default 0), `max_matches` (default 1,000, cap 10,000), `max_bytes` | `{matches, totalMatches, matchOffset, nextMatchOffset?, omittedMatches, truncated}` | -| `workspace_diff` | `change_offset` (default 0), `paths` (optional exact paths), `max_bytes`, `max_file_bytes` | `{changes, statistics, totalChanges, changeOffset, nextChangeOffset?, omittedFiles, omittedTextDiffs, truncated, unifiedPatch?}` | -| `workspace_apply` | `max_bytes`, `max_file_bytes` | `{changes, failures, statistics, totalChanges, changeOffset: 0, omittedFiles, omittedTextDiffs, truncated}` | - -Continue glob/diff at the returned offset, keeping the same pattern/paths and revisions. Pin both -diff revisions to checkpoints for stable paging while the workspace changes. Narrow glob patterns -or diff `paths`, or raise `max_file_bytes`, to retrieve detail; use `workspace_read` for full text. -`max_text_bytes` skips text diffing for files larger than its source-byte limit (default and cap -1,000,000 bytes per input file); `max_file_bytes` bounds encoded diff bytes per file in the result. -The nested `changes` contains only returned detail; top-level `statistics` describes all selected -changes using available computed text diffs. `omittedFiles` counts change entries at/after the -requested offset that are absent from this response, including entries on subsequent pages. -`omittedTextDiffs` counts returned entries whose computed diff was removed by the output budget. -Glob skips a single path too large for a page and increments `omittedMatches`; diff similarly -counts oversized path metadata in `omittedFiles`. Offsets still advance past those entries. - -Apply always completes the full atomic edit operation before bounding its response. Do not repeat -an apply call to retrieve omitted detail. Use `workspace_read` to inspect omitted files, or create -a checkpoint before applying and diff from it. If response encoding fails after commit, apply -returns a minimal success envelope with empty `changes`, full statistics, and `truncated: true`. +| `workspace_read` | Read UTF-8 text, current or from a checkpoint | `offset`; `max_bytes` of content (default 65,536, minimum 4, clamped to 262,144). Continue at `nextOffset`. | +| `workspace_glob` | List matching paths | `match_offset`; `max_matches` (default 1,000, clamped to 10,000); `max_bytes`. Continue at `nextMatchOffset`. | +| `workspace_search` | Literal or regex search | `match_offset`; `context_lines` (clamped to 100); `max_bytes`. Continue at `nextMatchOffset`. | +| `workspace_apply` | Apply `write_text`, `append_text`, `create_directory`, `remove`, `copy` and `move` edits atomically | `max_bytes`; `max_file_bytes` | +| `workspace_diff` | Compare checkpoints or the current state, optionally with a unified patch | `change_offset`; `paths`; `max_text_bytes`; `max_bytes`; `max_file_bytes`. Continue at `nextChangeOffset`. | +| `workspace_checkpoint` | Create a checkpoint | None | + +- `max_bytes` bounds the encoded JSON result of glob, search, diff and apply, including escaping: default 65,536, minimum 1,024, clamped to 262,144. +- `max_file_bytes` bounds each file's diff and patch text: default 8,192, clamped to 65,536; 0 omits all text. Omitted text has reason `outputBudgetExceeded` and is counted in `omittedTextDiffs`. `statistics` always describe the full change. +- `max_text_bytes` (default and maximum 1,000,000) skips text diffs for larger files. + +`workspace_apply` finishes the whole edit before trimming its response, so never repeat an apply to recover detail; read the files, or diff from a checkpoint taken first. Diff between checkpoints for stable paging while the tree changes. + +`call` throws `WorkspaceToolError` for an unknown tool or invalid arguments. Workspace and filesystem errors, such as a missing file or a denial, propagate unchanged. Turn both into tool-error results for the model. + +## Guarantees and limits + +### Atomicity and the operation gate + +An atomic `apply` snapshots every path an edit touches, including parents it creates, and restores them if an edit or the history write fails. If the restore also fails, it throws `WorkspaceRollbackError` carrying both errors. Commits and `restore(to:)` recover the same way. Recovery doesn't cover hard links to an edited file outside those paths, permission changes on symbolic links themselves, or a crash: recovery runs in the process. + +Operations that mutate or capture the tree run one at a time per Workspace instance, in arrival order. This operation gate covers `apply` and its conveniences, `preview`, `createCheckpoint`, `restore`, `beginTransaction`, commits, `compact`, `history()` and diffs against `.current`. In practice: + +- An edit never interleaves with another edit, checkpoint or commit on the same instance. +- A slow operation, such as the first checkpoint of a large tree, delays everything queued behind it. +- Reads of the current state aren't queued and can see an edit partly applied. +- Other instances, writes through `workspace.fileSystem` and other processes aren't serialized. A failed atomic apply restores its paths over any concurrent change to them. + +### Path identity + +A `WorkspacePath` is absolute and normalized: `.` is dropped, and `..` removes a component but can't climb above the root. Its identity is the exact UTF-8 bytes of its components. Equality, hashing, ordering, permission rules and quotas don't fold case or Unicode normalization, so precomposed and decomposed `/café` are different paths. + +`LocalFileSystem` requires the spelling you use to match the stored name byte for byte, even on volumes that treat other spellings as the same file. On case-insensitive APFS with `/Readme.md` stored: + +- reading `/readme.md` fails with `ENOENT`; +- writing or creating `/readme.md`, or renaming `/Readme.md` to `/README.md`, fails with `EEXIST` (rename through a temporary name instead); +- removing `/readme.md` does nothing. + +HFS+ and macOS's exFAT driver store names in decomposed form (NFD). Creating a precomposed (NFC) name that the volume would decompose fails with `EILSEQ`, and the entry is removed. Use the spelling a directory listing reports. Linux case-folding directories and case-insensitive XFS follow the same alias rules. Listing a directory that contains a name that isn't valid UTF-8 throws `WorkspaceError.invalidPath`. + +### Safety layer, not a sandbox + +What the wrappers do: + +- `LocalFileSystem` holds a descriptor for its root, opens each component without following links, and resolves links itself, up to 40 hops. An access that would leave the root fails with `WorkspaceError.invalidPath`. A link pointing outside can exist and be read with `readSymlink`, but not followed. +- `AuthorizedFileSystem` checks the path you named and the path it resolves to. +- `LimitedFileSystem` rejects a mutation before it runs. + +What they don't do: + +- Confine your process. Code in it, and anything it launches, can reach any file the user can. +- Stop other processes. A host process can move an open directory out of the root, change a file through a hard link from outside, or rename entries between an authorization check and the operation. +- Enforce quotas against writers that bypass the `LimitedFileSystem`. + +Run untrusted code in an OS sandbox or container, and use these wrappers inside it for policy and bookkeeping. + +### History store and durability + +`.memory` persistence keeps checkpoints, including file contents, and history in process memory until the process exits. + +`.directory(url)` keeps each workspace in `url//`, in a single on-disk format marked by `format.json`. An existing directory whose marker is missing or different fails with `WorkspaceError.storageCorrupted`, and the message names the history directory to delete. There's no migration; deleting it discards that workspace's checkpoints and history, not its files. + +Several instances and processes can share a store. Compaction only protects checkpoints in use within its own process, so coordinate compaction across processes. + +A checkpoint becomes visible only after its files and directories are synced with `fsync`. On Apple platforms the snapshot root and checkpoint record also use `F_FULLFSYNC` to flush the drive cache; volumes that don't support it keep the `fsync` guarantee. A published checkpoint survives a crash or power loss on storage that honors these requests, and a failed capture never publishes a checkpoint with missing content. Mutation history is appended without `fsync`, so its latest records can be lost on power failure. + +### Change detection + +With the default `checkpointCapture: .trustMetadata`, a file whose size, modification time and identity (when known) match the previous capture isn't re-read. This applies to directory-store checkpoints, transactions, and checkpoint-to-current diffs and restores. + +Reuse also requires the modification time to be more than two seconds older than the start of the previous capture, because coarse timestamps can hide a second write in the same tick. Files modified inside that window, and files without a modification time, are re-read. The cost is an extra read of recently written files, not a correctness risk. `InMemoryFileSystem` has exact timestamps and skips the window. + +The check can't see a write that keeps the size and restores an old modification time, as some sync and archive tools do, or clock skew on a network filesystem. Use `.verifyContents` to read every file instead. ## Testing @@ -600,61 +441,16 @@ swift build swift test --disable-xctest --enable-swift-testing ``` -CI runs Swift 6.2 tests on macOS and Linux. +CI runs the tests with Swift 6.2 on macOS and Linux. ## Benchmarks -The separate executable does not run in `swift test`. Run in release mode: +`WorkspaceBenchmarks` is a separate executable for macOS and Linux, not part of `swift test`. Run it in release mode and compare later runs against a saved report: ```sh -swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/small.json +swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/before.json swift run -c release WorkspaceBenchmarks --preset small --json Benchmarks/Results/after.json \ - --compare Benchmarks/Results/small.json -swift run -c release WorkspaceBenchmarks --preset medium --filter transaction,limitedWrite + --compare Benchmarks/Results/before.json ``` -Defaults: one warmup, five measured samples, a fresh child per scenario. Fast operations -use inner repetitions with per-operation times/counts; writes use independent prepared -stores (each `write1001` is exactly the 1,001st mutation). Fixtures have 1k/10k/100k -regular files: small ~20 MiB, medium ~200 MiB, large ~98 MiB with deep dependency nesting. -Allow roughly **2–5 minutes for small**, **10–30 minutes for medium**, and **well over -an hour for large** at defaults, depending on filesystem and watcher availability. -Fixture generation is excluded. `--fixture-dir PATH` generates/validates/reuses a -preserved fixture; otherwise temporary trees/stores are removed on completion or SIGINT. -`Benchmarks/Results/` is ignored. Use `--help` for filters, iteration counts and -deadlines (30 minutes per child by default). - -| Scenario | What's timed | Setup excluded | Counter blind spots | -| --- | --- | --- | --- | -| `writeText`, `apply50`, `limitedWrite` | One recorded write, 50 edits, quota-limited write with warm accounting | No-op preparation, file reset, Limited accounting priming | Checkpoint store; Limited metadata traversal is visible | -| `transaction` / `transactionFirst` | Begin, one draft write, commit | Preparation; initial begin/discard and baseline checkpoint for steady-state `transaction` only | Draft filesystem, checkpoint store | -| `preview` | One edit preview | Preparation | Draft filesystem | -| `checkpointFirst` / `checkpointIncremental` | First capture / capture after one change | Preparation; baseline and change for incremental | Checkpoint store | -| `diff`, `restore` | Checkpoint-to-current diff / restore after one change | Preparation, baseline, change | Checkpoint store | -| `readCheckpoint` / `readCheckpointWarm`, `readCurrent` | Cold / primed checkpoint index read; direct current read | Baseline; first read for warm variant | Checkpoint store | -| `search`, `glob` | Full absent-literal search / `**/*` glob | Preparation | Glob internals; returned `globMatches` is visible | -| `history1000` / `history1000Cached` | Fresh directory Workspace log load/decode / cached memory history | Seed 1,000 mutations; new reader for cold variant | Checkpoint store | -| `write1001` | Exactly the 1,001st recorded directory write | Preparation and 1,000 mutations | Checkpoint store | -| `eventsInitial`, `eventsExternal` | Enable observation / direct external write until its event arrives | Preparation; initial observation and drain for external | External write bypasses counter; recaptures are visible | - -The table shows median/p90 wall time, backend read bytes/calls, stat/list calls, entries, -glob matches, logical store bytes/files written, setup RSS high-water, operation increase -in that high-water, and held RSS delta. Held RSS is sampled with the result live. The -first-run column is the live-footprint figure: after the first run, allocator reuse hides -most of a result's memory, so the measured-sample summary (which excludes warmups) can be -near zero or slightly negative. Zero high-water growth can likewise mean allocator reuse. -Percentiles use linear interpolation between samples. Setup high-water includes preceding -samples in the same child; JSON retains each sample's values and process peak. -Darwin/Linux RSS units are normalized to bytes. Store write counts include newly created -empty lock files. - -Counters observe the `FileSystem` boundary, not syscalls. Store writes are estimated -outside timing from directory metadata: append growth or final size of new/replaced -files; transient/deleted files, repeated overwrites and OS write amplification are -invisible. Compare identical presets for regressions: timing ratios and exact backend -count deltas are printed. Watcher errors produce a nonzero exit while retaining successful -results; run outside a restrictive macOS sandbox to measure FSEvents. - -Scoped walks report `walk` and `walkEntry` counters; their content callbacks still count as -`readFile`, `readRange`, or `readSymlink`. `directoryEntries` includes visited descendants. -These are logical visits, not POSIX syscall counts; backend-internal glob walks remain opaque. +Compare the same preset on the same machine. `--compare` prints timing ratios and exact I/O count differences, `--filter` selects scenarios, and `--help` lists the options. `Benchmarks/Results/` is git-ignored. See [Benchmarks/README.md](Benchmarks/README.md) for presets, scenarios and counters. From fa8ff684ab41a67b8e30bd305a9f6638342c4ee0 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 22:01:19 -0700 Subject: [PATCH 28/34] Read directory metadata in bulk on Darwin Local traversal now reads each directory's entries and their metadata with getattrlistbulk instead of readdir plus one fstatat per child. Regular files and symlinks take their kind, size, mode, mtime, identity and link count from the bulk record; directories, mount points and special files keep fstatat. Bulk names are only trusted under spelling policies where the returned name is the stored name, and they still pass entry validation. The access mask can report an effective rather than stored mode: APFS drops set-ID bits when the owner is unknown, as on nosuid/noowners mounts. Mounts with MNT_NOSUID, MNT_IGNORE_OWNERSHIP or MNT_QUARANTINE, and entries whose owner is unknown, fall back to fstatat. Partial or malformed batches are discarded in favour of the fallback, and a strict walk still reports a child that vanishes after listing. Scripts/check-bulk-attributes.sh runs the parity test on a mounted volume; it passes on APFS, HFS+ and exFAT images. On a native paired A/B against the previous commit, directory-persisted incremental checkpoints are 11-16% faster and glob 20% faster, with identical I/O counts; search is unchanged. --- README.md | 8 + Scripts/check-bulk-attributes.sh | 13 + .../Workspace/FS/DarwinBulkAttributes.swift | 150 +++++++++ Sources/Workspace/FS/LocalFileSystem.swift | 58 +++- .../DarwinBulkAttributesTests.swift | 314 ++++++++++++++++++ 5 files changed, 531 insertions(+), 12 deletions(-) create mode 100755 Scripts/check-bulk-attributes.sh create mode 100644 Sources/Workspace/FS/DarwinBulkAttributes.swift create mode 100644 Tests/WorkspaceTests/DarwinBulkAttributesTests.swift diff --git a/README.md b/README.md index 7821eb3..9e23052 100644 --- a/README.md +++ b/README.md @@ -443,6 +443,14 @@ swift test --disable-xctest --enable-swift-testing CI runs the tests with Swift 6.2 on macOS and Linux. +On macOS, `LocalFileSystem` reads directory metadata in bulk where the volume reports it exactly, and uses per-file lookups otherwise. To check that both give the same results on a particular volume, mount a writable test volume (APFS, HFS+ or exFAT) and run: + +```bash +Scripts/check-bulk-attributes.sh --volume /Volumes/TestVolume +``` + +The script creates and removes its own test folder and prints which cases ran. + ## Benchmarks `WorkspaceBenchmarks` is a separate executable for macOS and Linux, not part of `swift test`. Run it in release mode and compare later runs against a saved report: diff --git a/Scripts/check-bulk-attributes.sh b/Scripts/check-bulk-attributes.sh new file mode 100755 index 0000000..6aed899 --- /dev/null +++ b/Scripts/check-bulk-attributes.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Creates and removes a private parity tree on an already-mounted writable volume. +set -eu +if [ "$#" -ne 2 ] || [ "$1" != "--volume" ] || [ ! -d "$2" ]; then + echo "Usage: $0 --volume /Volumes/TestVolume" >&2 + exit 2 +fi +WORKSPACE_TEST_VOLUME=$(cd "$2" && pwd) +export WORKSPACE_TEST_VOLUME +printf 'Bulk attribute parity volume: %s\n' "$WORKSPACE_TEST_VOLUME" +printf '%s\n' 'CASE lines report exercised/skipped features; RAW lines diagnose kernel modes independently of the production route.' +cd "$(dirname "$0")/.." +swift test --disable-sandbox --filter DarwinBulkAttributesTests.metadataAndStoredNamesMatchStat diff --git a/Sources/Workspace/FS/DarwinBulkAttributes.swift b/Sources/Workspace/FS/DarwinBulkAttributes.swift new file mode 100644 index 0000000..6a3a36c --- /dev/null +++ b/Sources/Workspace/FS/DarwinBulkAttributes.swift @@ -0,0 +1,150 @@ +#if canImport(Darwin) +import Darwin +import Foundation + +/// A directory's records are committed only after the complete enumeration succeeds. +/// Unsupported devices are remembered per LocalFileSystem; transient errors only +/// abandon this directory. No descriptors or entry metadata survive a walk. +final class DarwinBulkAttributes: @unchecked Sendable { + typealias Read = @Sendable (Int32, UnsafeMutablePointer, UnsafeMutableRawPointer, Int, UInt64) -> Int32 + typealias MountFlags = @Sendable (Int32) -> UInt32? + private let read: Read + private let mountFlags: MountFlags + private let lock = NSLock() + private var unsupported: Set = [] + + init(read: @escaping Read = { getattrlistbulk($0, $1, $2, $3, $4) }, + mountFlags: @escaping MountFlags = { fd in + var volume = statfs() + return fstatfs(fd, &volume) == 0 ? volume.f_flags : nil + }) { + self.read = read + self.mountFlags = mountFlags + } + + struct Record { + let name: String + // Directories (including mount points) and special files retain lstat. + let info: stat? + } + + private enum Failure: Error { case malformed, missingAttributes } + private struct VanishedEntry: Error { let code: Int } + + func entries(in fd: Int32, device: dev_t, skippingVanishedEntries: Bool = false) throws -> [Record]? { + lock.lock() + let disabled = unsupported.contains(device) + lock.unlock() + if disabled { return nil } + // ATTR_CMN_ACCESSMASK can describe effective rather than stored mode: + // set-ID bits are dropped when the reported owner is unknown, observed + // on APFS mounts with nosuid/noowners. Unknown owners on eligible mounts + // fall back per entry in parse(_:request:). + // Also exclude ownership overrides and quarantine conservatively. nodev + // only affects special-file operations (those entries already use stat). + // Cache exclusions, never eligibility: a later remount must be rechecked. + guard let flags = mountFlags(fd) else { return nil } + if flags & UInt32(MNT_NOSUID | MNT_IGNORE_OWNERSHIP | MNT_QUARANTINE) != 0 { + lock.lock() + unsupported.insert(device) + lock.unlock() + return nil + } + do { return try enumerate(fd) } + catch { + // A bulk record already listed this child. Restarting a strict walk + // could hide its disappearance, unlike the original readdir + lstat. + if let vanished = error as? VanishedEntry, !skippingVanishedEntries { + throw NSError(domain: NSPOSIXErrorDomain, code: vanished.code) + } + let code = (error as NSError).code + if error as? Failure == .missingAttributes || + ((error as NSError).domain == NSPOSIXErrorDomain && + [Int(ENOTSUP), Int(ENOSYS), Int(EINVAL)].contains(code)) { + lock.lock() + unsupported.insert(device) + lock.unlock() + } + return nil + } + } + + private func enumerate(_ fd: Int32) throws -> [Record] { + var request = attrlist() + request.bitmapcount = UInt16(ATTR_BIT_MAP_COUNT) + request.commonattr = UInt32(ATTR_CMN_RETURNED_ATTRS) | UInt32(ATTR_CMN_ERROR) | UInt32(ATTR_CMN_NAME) + request.commonattr |= UInt32(ATTR_CMN_DEVID | ATTR_CMN_OBJTYPE | ATTR_CMN_MODTIME) + request.commonattr |= UInt32(ATTR_CMN_OWNERID | ATTR_CMN_ACCESSMASK | ATTR_CMN_FILEID) + request.fileattr = UInt32(ATTR_FILE_LINKCOUNT | ATTR_FILE_DATALENGTH) + let buffer = UnsafeMutableRawPointer.allocate(byteCount: 64 * 1024, alignment: 8) + defer { buffer.deallocate() } + var records: [Record] = [] + while true { + let count = read(fd, &request, buffer, 64 * 1024, UInt64(FSOPT_PACK_INVAL_ATTRS)) + guard count >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + if count == 0 { return records } + var offset = 0 + for _ in 0..= 4, length <= 64 * 1024 - offset else { throw Failure.malformed } + let bytes = UnsafeRawBufferPointer(start: buffer.advanced(by: offset), count: length) + records.append(try parse(bytes, request: request)) + offset += length + } + } + } + + private func parse(_ bytes: UnsafeRawBufferPointer, request: attrlist) throws -> Record { + // attrlist fields are packed on 4-byte boundaries, even for 64-bit values. + var cursor = 4 + func take(_ type: T.Type) throws -> T { + guard MemoryLayout.size <= bytes.count - cursor else { throw Failure.malformed } + defer { cursor += MemoryLayout.size } + return bytes.loadUnaligned(fromByteOffset: cursor, as: T.self) + } + let returned = try take(attribute_set_t.self) + let error = try take(UInt32.self) + guard returned.commonattr & UInt32(ATTR_CMN_ERROR) != 0 else { throw Failure.missingAttributes } + if error == UInt32(ENOENT) || error == UInt32(ENOTDIR) { throw VanishedEntry(code: Int(error)) } + guard error == 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(error)) } + guard returned.commonattr & request.commonattr == request.commonattr else { throw Failure.missingAttributes } + let referenceOffset = cursor + let reference = try take(attrreference_t.self) + let device = try take(dev_t.self) + let type = try take(UInt32.self) + let modified = try take(timespec.self) + let owner = try take(uid_t.self) + let mode = try take(UInt32.self) + let inode = try take(UInt64.self) + // File-group fields are absent for directories even with PACK_INVAL_ATTRS. + let links = type == 2 ? 0 : try take(UInt32.self) + let size = type == 2 ? 0 : try take(off_t.self) + let start = referenceOffset + Int(reference.attr_dataoffset) + let length = Int(reference.attr_length) + guard start >= cursor, length > 1, start <= bytes.count, + length <= bytes.count - start, bytes[start + length - 1] == 0 else { throw Failure.malformed } + let nameBytes = bytes[start..<(start + length - 1)] + guard !nameBytes.contains(0), !nameBytes.contains(47), + let name = String(bytes: nameBytes, encoding: .utf8), name != ".", name != ".." else { + throw Failure.malformed + } + // VREG=1, VDIR=2, VLNK=5 (sys/vnode.h). Other types have different + // size semantics, and bulk mount-point attributes describe the covered inode. + guard type == 1 || type == 5 else { return Record(name: name, info: nil) } + // KAUTH_UID_UNKNOWN (99): the access mask may omit stored set-ID bits. + guard owner != 99 else { return Record(name: name, info: nil) } + guard returned.fileattr & request.fileattr == request.fileattr, + let linkCount = nlink_t(exactly: links), modified.tv_nsec >= 0, + modified.tv_nsec < 1_000_000_000 else { throw Failure.missingAttributes } + var info = stat() + info.st_dev = device + info.st_ino = ino_t(inode) + info.st_mode = mode_t(mode & 0o7777) | (type == 1 ? S_IFREG : S_IFLNK) + info.st_mtimespec = modified + info.st_nlink = linkCount + info.st_size = size + return Record(name: name, info: info) + } +} +#endif diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index 95c11ff..ecd7a96 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -23,6 +23,17 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem /// so unsupported filesystem behavior can be tested without mounts or global syscall hooks. typealias ExclusiveRename = @Sendable (Int32, String, Int32, String) -> Int32 + #if canImport(Darwin) + private var bulkAttributes: DarwinBulkAttributes? = DarwinBulkAttributes() + + // Per-instance route/syscall injection for parity and fallback validation. + convenience init(root: URL, bulkAttributes: DarwinBulkAttributes?) throws { + self.init() + self.bulkAttributes = bulkAttributes + try applyConfiguration(rootDirectory: root) + } + #endif + private let fileManager: FileManager private let spelling = LocalPathSpelling() private let exclusiveRename: ExclusiveRename @@ -105,8 +116,8 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem return entry } - // Names emitted by readdir already have stored spelling. This initializer - // still applies the jail's byte-level single-component validation. + // Readdir and supported native bulk enumeration supply stored spelling. + // This initializer still applies the jail's single-component validation. init(parent: Descriptor, storedName: String, path: WorkspacePath) throws { try self.init(parent: parent, name: storedName, path: path) isVerified = true @@ -599,16 +610,8 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem let children: [(entry: Entry, info: LocalStat, key: String)] do { directory = try openEntry(entry, flags: O_RDONLY | O_DIRECTORY) - children = try names(in: directory, consumingOffset: true).compactMap { name -> (entry: Entry, info: LocalStat, key: String)? in - let child = try Entry(parent: directory, storedName: name, path: entry.path.appending(name)) - let info: LocalStat - do { info = try self.metadata(child) } - catch { - guard skippingVanishedEntries, Snapshot.isMissingEntry(error) else { throw error } - return nil - } - return (child, info, walkSortKey(name: name, isDirectory: info.st_mode & S_IFMT == S_IFDIR)) - }.sorted { $0.key.utf8.lexicographicallyPrecedes($1.key.utf8) } + children = try walkChildren(in: directory, at: entry.path, + skippingVanishedEntries: skippingVanishedEntries) } catch { guard skippingUnreadableDirectories, isSkippableWalkError(error, at: entry.path) else { throw error } try await visit(metadata, { _ in throw WorkspaceError.unsupported("cannot read a directory") }, @@ -624,6 +627,37 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem { try self.linkTarget(entry) }) } + private func walkChildren(in directory: Descriptor, at path: WorkspacePath, + skippingVanishedEntries: Bool) throws -> [(entry: Entry, info: LocalStat, key: String)] { + var records: [(name: String, info: LocalStat?)]? + var consumedOffset = false + #if canImport(Darwin) + // Only APFS/HFS supply authoritative stored names through native bulk + // enumeration. Unknown/FSKit/network volumes keep readdir as their source. + if let bulkAttributes, let device = directory.directoryInfo?.st_dev, + directory.spellingPolicy == .darwin || directory.spellingPolicy == .hfs { + consumedOffset = true + records = try bulkAttributes.entries(in: directory.value, device: device, + skippingVanishedEntries: skippingVanishedEntries)?.map { ($0.name, $0.info) } + } + #endif + if records == nil { + // A failed bulk call can have advanced the fd, even after valid batches. + // Reopen "." for an independent offset; never resume a partial listing. + records = try names(in: directory, consumingOffset: !consumedOffset).map { ($0, nil) } + } + return try (records ?? []).compactMap { name, suppliedInfo in + let child = try Entry(parent: directory, storedName: name, path: path.appending(name)) + let info: LocalStat + do { info = try suppliedInfo ?? self.metadata(child) } + catch { + guard skippingVanishedEntries, Snapshot.isMissingEntry(error) else { throw error } + return nil + } + return (child, info, walkSortKey(name: name, isDirectory: info.st_mode & S_IFMT == S_IFDIR)) + }.sorted { $0.key.utf8.lexicographicallyPrecedes($1.key.utf8) } + } + private func openRegularFile(_ entry: Entry) throws -> (Descriptor, LocalStat) { let fd = try openEntry(entry, flags: O_RDONLY | O_NONBLOCK) var info = LocalStat() diff --git a/Tests/WorkspaceTests/DarwinBulkAttributesTests.swift b/Tests/WorkspaceTests/DarwinBulkAttributesTests.swift new file mode 100644 index 0000000..4b5532c --- /dev/null +++ b/Tests/WorkspaceTests/DarwinBulkAttributesTests.swift @@ -0,0 +1,314 @@ +#if canImport(Darwin) +import Darwin +import Foundation +import Testing +@testable import Workspace + +struct DarwinBulkAttributesTests { + private func root() throws -> URL { + let parent = ProcessInfo.processInfo.environment["WORKSPACE_TEST_VOLUME"].map { URL(fileURLWithPath: $0) } + ?? URL(fileURLWithPath: "/private/tmp") + let root = parent.appendingPathComponent("WorkspaceBulkParity-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + return root + } + + private func infos(_ fs: LocalFileSystem, skipUnreadable: Bool = true, skipVanished: Bool = false) async throws -> [FileInfo] { + var result: [FileInfo] = [] + try await fs.walk(at: .root, skippingUnreadableDirectories: skipUnreadable, + skippingVanishedEntries: skipVanished) { info, _, readLink in + result.append(info) + if info.kind == .symlink { _ = try await readLink() } + } + return result + } + + private func equal(_ actual: [FileInfo], _ expected: [FileInfo]) throws { + #expect(actual.count == expected.count, "visit count: bulk=\(actual.count), stat=\(expected.count)") + for (index, pair) in zip(actual, expected).enumerated() { + let (bulk, stat) = pair + // Evaluate and format outside #expect so failures never hide values + // behind a throwing expression's diagnostic. + var differences: [String] = [] + func compare(_ field: String, _ a: T, _ b: T) { + if a != b { differences.append("\(field): bulk=\(a), stat=\(b)") } + } + compare("path UTF-8/order", Array(bulk.path.string.utf8), Array(stat.path.string.utf8)) + compare("kind", bulk.kind, stat.kind) + compare("size", bulk.size, stat.size) + compare("permissions (octal)", String(bulk.permissions.rawValue, radix: 8), String(stat.permissions.rawValue, radix: 8)) + compare("modificationDate", bulk.modificationDate?.timeIntervalSinceReferenceDate, stat.modificationDate?.timeIntervalSinceReferenceDate) + compare("identity", bulk.identity, stat.identity) + compare("linkCount", bulk.linkCount, stat.linkCount) + let detail = "visit \(index) \(stat.path): " + differences.joined(separator: "; ") + #expect(differences.isEmpty, Comment(rawValue: detail)) + } + } + + @Test func metadataAndStoredNamesMatchStat() async throws { + let root = try root() + defer { TestSupport.remove(root) } + let manager = FileManager.default + let nativeVolume = ProcessInfo.processInfo.environment["WORKSPACE_TEST_VOLUME"] != nil + func report(_ name: String, result: Int32) { + let code = errno + if result == 0 { print("CASE \(name): ran") } + else { + print("CASE \(name): SKIPPED (errno \(code): \(String(cString: strerror(code))))") + #expect(nativeVolume, "Default fixture must support \(name)") + } + } + for name in ["empty", "nested", "unreadable"] { + try manager.createDirectory(at: root.appendingPathComponent(name), withIntermediateDirectories: false) + } + let files: [(String, mode_t)] = [ + ("regular", 0o640), ("nested/file", 0o755), ("NFC-é", 0o444), ("NFD-e\u{301}", 0o600), + ("mode-special", 0o6751), ("mode-setuid", 0o4751), ("mode-setgid", 0o2751), + ("mode-sticky", 0o1751), ("mode-all-special", 0o7751), + ] + // POSIX creation preserves supplied Unicode bytes on APFS; Foundation + // URLs can decompose them before creation. HFS+ rewrites them on disk. + for (index, file) in files.enumerated() { + let (name, mode) = file + let path = root.path + "/" + name + let fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0o600) + try #require(fd >= 0) + defer { close(fd) } + let data = Data(repeating: UInt8(index), count: index * 17 + 1) + #expect(data.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) } == data.count) + let changed = fchmod(fd, mode) + let chmodError = errno + var status = stat() + #expect(fstat(fd, &status) == 0) + let actualMode = status.st_mode & 0o7777 + let supported = changed == 0 && actualMode == mode + print("CASE \(name) mode: \(supported ? "ran" : "SKIPPED requested mode") requested=\(String(mode, radix: 8)) stat=\(String(actualMode, radix: 8)) chmodErrno=\(changed == 0 ? 0 : chmodError)") + #expect(nativeVolume || (changed == 0 && actualMode & 0o1777 == mode & 0o1777), + "Default fixture must retain ordinary and sticky mode bits for \(name)") + var times = [timespec(tv_sec: 1_700_000_000, tv_nsec: 123_456_789), + timespec(tv_sec: 1_700_000_000, tv_nsec: 987_654_321)] + #expect(utimensat(AT_FDCWD, path, ×, 0) == 0) + } + try Data().write(to: root.appendingPathComponent("zero-bytes")) + print("CASE empty/nested directories, zero-byte file, Unicode spellings, fractional mtime: ran") + // Probe each feature independently: a lack of hard links says nothing + // about whether symlinks, FIFOs, or sockets can be represented. + report("hard link", result: link(root.appendingPathComponent("regular").path, root.appendingPathComponent("hard").path)) + for (name, target) in [("dangling", "missing"), ("cycle", "cycle"), ("escape", "/etc/passwd"), + ("dir-link", "nested"), ("file-link", "regular")] { + report("symlink \(name)", result: symlink(target, root.appendingPathComponent(name).path)) + } + report("FIFO", result: mkfifo(root.appendingPathComponent("fifo").path, 0o620)) + let socketFD = socket(AF_UNIX, SOCK_STREAM, 0) + try #require(socketFD >= 0) + defer { close(socketFD) } + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let socketPath = Array(root.appendingPathComponent("socket").path.utf8CString) + if socketPath.count <= MemoryLayout.size(ofValue: address.sun_path) { + withUnsafeMutableBytes(of: &address.sun_path) { bytes in + for (index, byte) in socketPath.enumerated() { bytes[index] = UInt8(bitPattern: byte) } + } + let result = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(socketFD, $0, socklen_t(MemoryLayout.size)) } + } + report("socket", result: result) + } else { print("CASE socket: SKIPPED (volume path exceeds sockaddr_un capacity)") } + let unreadable = root.appendingPathComponent("unreadable") + #expect(chmod(unreadable.path, 0) == 0) + defer { _ = chmod(unreadable.path, 0o700) } + print("CASE unreadable directory: \(access(unreadable.path, R_OK) == 0 ? "SKIPPED denial (still readable)" : "ran")") + for alias in ["REGULAR", "NFC-e\u{301}", "NFD-é"] { + var status = stat() + _ = lstat(root.appendingPathComponent(alias).path, &status) + } + let bulk = try LocalFileSystem(root: root, bulkAttributes: DarwinBulkAttributes()) + let reference = try LocalFileSystem(root: root, bulkAttributes: nil) + try equal(await infos(bulk), await infos(reference)) + let fd = open(root.path, O_RDONLY | O_DIRECTORY) + try #require(fd >= 0) + defer { close(fd) } + var status = stat() + #expect(fstat(fd, &status) == 0) + var volume = statfs() + #expect(fstatfs(fd, &volume) == 0) + let typeCapacity = MemoryLayout.size(ofValue: volume.f_fstypename) + let type = withUnsafePointer(to: &volume.f_fstypename) { + $0.withMemoryRebound(to: CChar.self, capacity: typeCapacity) { String(cString: $0) } + } + let listed = try await reference.listDirectory(path: .root) + let names = Set(listed.map { Data($0.name.utf8) }) + if type == "apfs" { + #expect(names.contains(Data("NFC-é".utf8))) + #expect(names.contains(Data("NFD-e\u{301}".utf8))) + } else if type == "hfs" { + #expect(names.contains(Data("NFC-e\u{301}".utf8))) + } + print("Volume type: \(type), flags: 0x\(String(volume.f_flags, radix: 16))") + let records = try DarwinBulkAttributes().entries(in: fd, device: status.st_dev) + let nativeNames = type == "apfs" || type == "hfs" + print("Production route: \(nativeNames && records != nil ? "bulk" : "readdir + fstatat (mount flags, filesystem, or unsupported attributes)")") + if !nativeVolume { #expect(records != nil) } + // Diagnostic only: explicitly bypass mount eligibility on a fresh fd to + // show what the kernel masks, without feeding raw metadata to a walk. + let rawFD = openat(fd, ".", O_RDONLY | O_DIRECTORY) + try #require(rawFD >= 0) + defer { close(rawFD) } + let raw = try DarwinBulkAttributes(mountFlags: { _ in 0 }).entries(in: rawFD, device: status.st_dev) + if let raw { + #expect(Set(raw.map { Data($0.name.utf8) }) == names) + #expect(raw.first { $0.name == "regular" }?.info != nil) + for record in raw { + if record.name.hasPrefix("NFC-") || record.name.hasPrefix("NFD-") { + print("Bulk/readdir stored bytes: \(Array(record.name.utf8))") + } + if record.name.hasPrefix("mode-"), let info = record.info { + var expected = stat() + #expect(fstatat(fd, record.name, &expected, AT_SYMLINK_NOFOLLOW) == 0) + let a = info.st_mode & 0o7777, b = expected.st_mode & 0o7777 + print("RAW \(record.name): bulk=\(String(a, radix: 8)) stat=\(String(b, radix: 8)) differingBits=\(String(a ^ b, radix: 8))") + } + } + } else { print("RAW bulk probe: unavailable; fallback parity only") } + for fs in [bulk, reference] { + do { + _ = try await infos(fs, skipUnreadable: false) + #expect(geteuid() == 0 || access(unreadable.path, R_OK) == 0) + } catch { #expect((error as NSError).code == Int(EACCES)) } + } + } + + @Test(arguments: [UInt32(0), UInt32(MNT_NODEV | MNT_NOEXEC | MNT_RDONLY | MNT_DONTBROWSE), + UInt32(MNT_NOSUID), UInt32(MNT_IGNORE_OWNERSHIP), UInt32(MNT_QUARANTINE), + UInt32(MNT_NOSUID | MNT_NODEV | MNT_IGNORE_OWNERSHIP | MNT_DONTBROWSE), nil]) + func mountFlagsPreserveStoredModes(flags: UInt32?) async throws { + let root = try root() + defer { TestSupport.remove(root) } + try Data([1]).write(to: root.appendingPathComponent("mode-special")) + #expect(chmod(root.appendingPathComponent("mode-special").path, 0o7751) == 0) + // The sandbox silently strips set-ID bits even after successful chmod. + // Inject loss of all special bits, including the retained sticky bit, + // so a missed gate corrupts FileInfo even here, not just a call count. + let allowed = flags == 0 || flags == UInt32(MNT_NODEV | MNT_NOEXEC | MNT_RDONLY | MNT_DONTBROWSE) + let probe = MountFlagProbe(flags: flags, stripSpecialBits: !allowed) + let bulk = try LocalFileSystem(root: root, bulkAttributes: DarwinBulkAttributes(read: probe.read, mountFlags: probe.flags)) + let reference = try LocalFileSystem(root: root, bulkAttributes: nil) + let expected = try await infos(reference) + let storedMode = try #require(expected.first { $0.path.string == "/mode-special" }).permissions.rawValue + try #require(storedMode & 0o1000 != 0) + for _ in 0..<2 { try equal(await infos(bulk), expected) } + #expect(allowed ? probe.reads > 0 : probe.reads == 0) + // Unsafe devices remain excluded; failed flag queries retry next time. + #expect(flags != nil && !allowed ? probe.queries == 1 : probe.queries == 2) + } + + @Test(arguments: ["io", "unsupported", "missing", "malformed", "vanishedStrict", "vanishedSkip"]) + func discardsPartialBatchesAndPreservesFallback(failure: String) async throws { + let root = try root() + defer { TestSupport.remove(root) } + for index in 0..<900 { + try Data([1]).write(to: root.appendingPathComponent("file-\(index)")) + } + let probe = FailingBulkRead(failure) + let bulk = try LocalFileSystem(root: root, bulkAttributes: DarwinBulkAttributes(read: probe.read)) + let reference = try LocalFileSystem(root: root, bulkAttributes: nil) + let expected: [FileInfo] + if failure.hasPrefix("vanished") { + if failure == "vanishedStrict" { + do { + _ = try await infos(bulk) + Issue.record("A child listed by bulk enumeration disappeared before metadata") + } catch { #expect((error as NSError).code == Int(ENOENT)) } + expected = try await infos(reference) + } else { + let initialRoot = try await reference.stat(path: .root) + let actual = try await infos(bulk, skipVanished: true) + expected = try await infos(reference) + // The root was inspected before the injected deletion; its size + // and mtime legitimately precede the surviving child metadata. + var duringDeletion = expected + duringDeletion[duringDeletion.count - 1] = initialRoot + try equal(actual, duringDeletion) + } + #expect(expected.count == 900) // 899 surviving files and the root + } else { + expected = try await infos(reference) + try equal(await infos(bulk), expected) + } + let firstCalls = probe.calls + #expect(firstCalls >= 2) // one real, nonempty batch before failure + try equal(await infos(bulk), expected) + if failure == "unsupported" || failure == "missing" { #expect(probe.calls == firstCalls) } + else { #expect(probe.calls > firstCalls) } + } +} + +private final class MountFlagProbe: @unchecked Sendable { + private let lock = NSLock() + private var readCount = 0 + private var queryCount = 0 + private let value: UInt32? + private let stripSpecialBits: Bool + init(flags: UInt32?, stripSpecialBits: Bool) { value = flags; self.stripSpecialBits = stripSpecialBits } + var reads: Int { lock.withLock { readCount } } + var queries: Int { lock.withLock { queryCount } } + func flags(_ fd: Int32) -> UInt32? { + lock.withLock { queryCount += 1 } + return value + } + func read(_ fd: Int32, _ attributes: UnsafeMutablePointer, _ buffer: UnsafeMutableRawPointer, + _ size: Int, _ options: UInt64) -> Int32 { + lock.withLock { readCount += 1 } + let count = getattrlistbulk(fd, attributes, buffer, size, options) + if stripSpecialBits, count > 0 { + var offset = 0 + for _ in 0...size + 4 + MemoryLayout.size + + MemoryLayout.size + 4 + MemoryLayout.size + let mode = buffer.loadUnaligned(fromByteOffset: modeOffset, as: UInt32.self) + buffer.storeBytes(of: mode & ~UInt32(S_ISUID | S_ISGID | S_ISVTX), toByteOffset: modeOffset, as: UInt32.self) + offset += Int(buffer.loadUnaligned(fromByteOffset: offset, as: UInt32.self)) + } + } + return count + } +} + +private final class FailingBulkRead: @unchecked Sendable { + private let lock = NSLock() + private var count = 0 + let failure: String + init(_ failure: String) { self.failure = failure } + var calls: Int { lock.withLock { count } } + func read(_ fd: Int32, _ attributes: UnsafeMutablePointer, _ buffer: UnsafeMutableRawPointer, + _ size: Int, _ options: UInt64) -> Int32 { + let call = lock.withLock { count += 1; return count } + if call == 2 { + if failure == "io" || failure == "unsupported" { + errno = failure == "io" ? EIO : ENOTSUP + return -1 + } + let result = getattrlistbulk(fd, attributes, buffer, size, options) + if result > 0 { + if failure.hasPrefix("vanished") { + // ATTR_CMN_ERROR follows the returned-attribute set; the next + // field is the name reference. Remove the actual listed child. + let referenceOffset = 4 + MemoryLayout.size + 4 + let reference = buffer.loadUnaligned(fromByteOffset: referenceOffset, as: attrreference_t.self) + let name = buffer.advanced(by: referenceOffset + Int(reference.attr_dataoffset)).assumingMemoryBound(to: CChar.self) + #expect(unlinkat(fd, name, 0) == 0) + buffer.storeBytes(of: UInt32(ENOENT), toByteOffset: 4 + MemoryLayout.size, as: UInt32.self) + } else if failure == "malformed" { buffer.storeBytes(of: UInt32.max, as: UInt32.self) } + else { + var returned = buffer.loadUnaligned(fromByteOffset: 4, as: attribute_set_t.self) + returned.commonattr &= ~UInt32(ATTR_CMN_FILEID) + buffer.storeBytes(of: returned, toByteOffset: 4, as: attribute_set_t.self) + } + } + return result + } + return getattrlistbulk(fd, attributes, buffer, size, options) + } +} +#endif From 57dc8423196874abc4fe4179cbcdbca9143a4576 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 22:07:43 -0700 Subject: [PATCH 29/34] Prune history at compaction, sync it at checkpoints, and page it Compaction now removes history records that every retained checkpoint already includes: those at or before the smallest retained mutation cursor, also with `.all`. With no checkpoints, history is kept. The log is validated before compaction changes any checkpoint or blob, then rewritten under the mutations lock into a synced replacement that is renamed into place. The sequence high-water mark and a cache epoch are durably bumped first, so sequence numbers are never reused and other instances drop stale caches. CompactionReport gains removedMutationCount and removedMutationBytes; dry runs project them. Every checkpoint publication path (create, transaction commit, restore and compaction rebases) syncs the history log and sequence sidecar first, with F_FULLFSYNC on Darwin, so a published checkpoint never refers to history a crash can lose. Ordinary appends stay unsynced. history(after:limit:) returns records oldest first and history(before:limit:) newest first, with exclusive cursors. Pages binary-search the sorted log on sequence numbers alone and decode only the records they return: a five-record page of a 1,000-record log does five full decodes. Negative limits throw. --- README.md | 20 +- .../Workspace/CheckpointStore+Revision.swift | 2 +- Sources/Workspace/CheckpointStore.swift | 515 +++++++++++++----- Sources/Workspace/WorkspaceCompaction.swift | 14 +- Sources/Workspace/WorkspaceHistory.swift | 21 +- .../WorkspaceTests/CheckpointStoreTests.swift | 2 +- Tests/WorkspaceTests/CompactionTests.swift | 2 +- .../HistoryDurabilityTests.swift | 137 +++++ Tests/WorkspaceTests/HistoryTests.swift | 349 ++++++++++++ 9 files changed, 923 insertions(+), 139 deletions(-) create mode 100644 Tests/WorkspaceTests/HistoryDurabilityTests.swift create mode 100644 Tests/WorkspaceTests/HistoryTests.swift diff --git a/README.md b/README.md index 9e23052..8482f7e 100644 --- a/README.md +++ b/README.md @@ -174,11 +174,14 @@ let plan = try await workspace.compact( dryRun: true ) print(plan.removedCheckpointCount, plan.reclaimedBytes) +print(plan.removedMutationCount, plan.removedMutationBytes) let report = try await workspace.compact(retaining: .latest(50)) ``` -`.latest` keeps the newest checkpoints, labeled ones and any IDs you list. It also keeps the checkpoint each kept rollback or transaction checkpoint came from, and any checkpoint that an open transaction or another Workspace instance in the process depends on. `.all` keeps every checkpoint and removes only unreferenced data. Survivors are rebased onto their nearest kept ancestor. `storageStatistics()` reports usage. Nothing prunes mutation history, so use `recording` to bound it. +`.latest` keeps the newest checkpoints, labeled ones and any IDs you list. It also keeps the checkpoint each kept rollback or transaction checkpoint came from, and any checkpoint that an open transaction or another Workspace instance in the process depends on. `.all` keeps every checkpoint. Survivors are rebased onto their nearest kept ancestor. `storageStatistics()` reports checkpoint data usage. + +Compaction also removes history records that every kept checkpoint already includes, even with `.all`. With no checkpoints, all history is kept. Sequence numbers are never reused. `removedMutationCount` and `removedMutationBytes` report what was removed, and dry runs project them. `reclaimedBytes` does not include history. ## Transactions @@ -253,12 +256,17 @@ let watcher = Task { } } -let mutations = try await workspace.history() +let mutations = try await workspace.history() // retained records, oldest first +let page = try await workspace.history(after: 100, limit: 50) // sequences > 100, oldest first +let newest = try await workspace.history(before: nil, limit: 50) // latest 50, newest first +let older = try await workspace.history(before: newest.last?.sequence, limit: 50) ``` `.changes` events cover edits, restores, archive restores and commits made through this Workspace, plus external changes the filesystem reports. Events don't say which. `.checkpoint` events cover this instance's checkpoints and, with `.directory` persistence, those of other instances sharing the store. Cancel the consuming task to end a stream. -`history()` returns `Mutation` records for edits, rollbacks, transactions and archive restores from every Workspace sharing the store. It waits for this instance's in-flight mutations. External changes never enter history. +`history()` returns all retained `Mutation` records for edits, rollbacks, transactions and archive restores from every Workspace sharing the store, oldest first. Every history call waits for this instance's in-flight mutations. External changes never enter history. + +`history(after:limit:)` returns up to `limit` records with a sequence greater than `after`, oldest first. `history(before:limit:)` returns up to `limit` records with a sequence less than `before`, newest first; pass `nil` to start from the newest. Cursors are exclusive, and there can be gaps. A zero limit returns an empty page; a negative limit throws. Each call reads history as it is at that moment, so a series of pages is not a snapshot. ### External changes @@ -381,7 +389,7 @@ Each definition's `inputSchema` is a `JSONValue` holding a plain JSON Schema obj An atomic `apply` snapshots every path an edit touches, including parents it creates, and restores them if an edit or the history write fails. If the restore also fails, it throws `WorkspaceRollbackError` carrying both errors. Commits and `restore(to:)` recover the same way. Recovery doesn't cover hard links to an edited file outside those paths, permission changes on symbolic links themselves, or a crash: recovery runs in the process. -Operations that mutate or capture the tree run one at a time per Workspace instance, in arrival order. This operation gate covers `apply` and its conveniences, `preview`, `createCheckpoint`, `restore`, `beginTransaction`, commits, `compact`, `history()` and diffs against `.current`. In practice: +Operations that mutate or capture the tree run one at a time per Workspace instance, in arrival order. This operation gate covers `apply` and its conveniences, `preview`, `createCheckpoint`, `restore`, `beginTransaction`, commits, `compact`, all `history` overloads and diffs against `.current`. In practice: - An edit never interleaves with another edit, checkpoint or commit on the same instance. - A slow operation, such as the first checkpoint of a large tree, delays everything queued behind it. @@ -424,7 +432,9 @@ Run untrusted code in an OS sandbox or container, and use these wrappers inside Several instances and processes can share a store. Compaction only protects checkpoints in use within its own process, so coordinate compaction across processes. -A checkpoint becomes visible only after its files and directories are synced with `fsync`. On Apple platforms the snapshot root and checkpoint record also use `F_FULLFSYNC` to flush the drive cache; volumes that don't support it keep the `fsync` guarantee. A published checkpoint survives a crash or power loss on storage that honors these requests, and a failed capture never publishes a checkpoint with missing content. Mutation history is appended without `fsync`, so its latest records can be lost on power failure. +A checkpoint becomes visible only after its files and directories are synced with `fsync`. On Apple platforms the snapshot root and checkpoint record also use `F_FULLFSYNC` to flush the drive cache; volumes that don't support it keep the `fsync` guarantee. A published checkpoint survives a crash or power loss on storage that honors these requests, and a failed capture never publishes a checkpoint with missing content. + +Mutation appends aren't synced, for speed. Publishing any checkpoint first syncs the history log, using `F_FULLFSYNC` on Apple platforms with the same `fsync` fallback. A published checkpoint never refers to history that a crash can lose. Records since your latest checkpoint can be lost on power failure. Compaction replaces the log atomically. ### Change detection diff --git a/Sources/Workspace/CheckpointStore+Revision.swift b/Sources/Workspace/CheckpointStore+Revision.swift index a78b2df..17459dc 100644 --- a/Sources/Workspace/CheckpointStore+Revision.swift +++ b/Sources/Workspace/CheckpointStore+Revision.swift @@ -180,7 +180,7 @@ extension FileCheckpointStore { // All immutable objects are durable before the manifest, then the checkpoint // is published through the same full-sync/fallback path as captureRevision. try write(manifest, to: snapshotURL(id: manifest.id, workspaceId: workspaceId), durability: .publication) - try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: workspaceId), durability: .publication) + try await publishCheckpoint(checkpoint) listCheckpointsCache[workspaceId] = nil return checkpoint } diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index a286d5a..a468439 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -6,6 +6,13 @@ import Darwin import Glibc #endif +// Both stores use the on-disk compact representation for history byte accounting. +private let compactStoreEncoder: JSONEncoder = { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return encoder +}() + /// Persistence for workspace checkpoints, snapshots, and mutation logs. /// /// The store is the **source of truth** for `Mutation.sequence` values. Callers may pass @@ -32,11 +39,11 @@ protocol CheckpointStore: CheckpointRevisionStore, AnyObject, Sendable { @discardableResult func appendMutation(_ mutation: Mutation) async throws -> Mutation func listMutations(workspaceId: UUID, afterSequence: Int) async throws -> MutationPage + func listMutations(workspaceId: UUID, query: MutationQuery) async throws -> MutationPage /// Retracts one failed operation's record without lowering the workspace's /// sequence high-water mark. Later appends must never reuse its sequence. func removeMutation(sequence: Int, workspaceId: UUID) async throws - /// Removes mutation records with `sequence <= throughSequence`. The record carrying the - /// highest sequence is always retained so later appends stay monotonic. + /// Removes mutation records with `sequence <= throughSequence`, preserving the high-water mark. func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws func storageStatistics(workspaceId: UUID) async throws -> Workspace.StorageStatistics func compact( @@ -52,7 +59,46 @@ struct MutationPage: Sendable { var retractionEpoch: Int } +enum MutationQuery: Sendable { + case after(Int, limit: Int) + case before(Int?, limit: Int) + + var limit: Int { + switch self { + case let .after(_, limit), let .before(_, limit): limit + } + } + + func validate() throws { + guard limit >= 0 else { throw WorkspaceError.unsupported("history limit must not be negative") } + } + + func select(_ records: [Mutation]) -> [Mutation] { + guard limit > 0 else { return [] } + switch self { + case let .after(cursor, limit): + return Array(records.lazy.filter { $0.sequence > cursor }.prefix(limit)) + case let .before(cursor, limit): + return Array(records.reversed().lazy.filter { record in + cursor.map { record.sequence < $0 } ?? true + }.prefix(limit)) + } + } +} + +struct MutationPruningResult { + var count = 0 + var bytes: UInt64 = 0 +} + extension CheckpointStore { + func listMutations(workspaceId: UUID, query: MutationQuery) async throws -> MutationPage { + try query.validate() + var page = try await listMutations(workspaceId: workspaceId, afterSequence: 0) + page.records = query.select(page.records) + return page + } + func resolveSnapshotFile(id: UUID, workspaceId: UUID, path: WorkspacePath) async throws -> (path: WorkspacePath, size: UInt64) { guard let index = try await loadRevisionIndex(id: id, workspaceId: workspaceId) else { throw WorkspaceError.revisionDataNotFound(id) @@ -219,14 +265,8 @@ actor InMemoryCheckpointStore: CheckpointStore { } func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws { - let records = (mutationsByWorkspace[workspaceId] ?? []).sorted { $0.sequence < $1.sequence } - guard let last = records.last else { - return - } - var kept = records.filter { $0.sequence > throughSequence } - if kept.isEmpty { - kept = [last] - } + let records = mutationsByWorkspace[workspaceId] ?? [] + let kept = records.filter { $0.sequence > throughSequence } if kept.count != records.count { mutationRetractionEpochs[workspaceId, default: 0] += 1 } mutationsByWorkspace[workspaceId] = kept } @@ -254,15 +294,23 @@ actor InMemoryCheckpointStore: CheckpointStore { let retainedSnapshotIDs = Set(retained.map(\.snapshotId)) let retainedSnapshots = currentSnapshots.filter { retainedSnapshotIDs.contains($0.key) } let after = statistics(checkpoints: retained, snapshots: retainedSnapshots) + let cutoff = retained.map(\.mutationCursor).min() + let pruned = (mutationsByWorkspace[workspaceId] ?? []).filter { record in + cutoff.map { record.sequence <= $0 } ?? false + } + let mutationBytes = try pruned.reduce(UInt64(0)) { $0 + UInt64(try compactStoreEncoder.encode($1).count + 1) } let report = Workspace.CompactionReport( dryRun: dryRun, before: before, after: after, removedCheckpointIDs: plan.removedCheckpointIDs, - rebasedCheckpointIDs: plan.rebasedCheckpointIDs + rebasedCheckpointIDs: plan.rebasedCheckpointIDs, + removedMutationCount: pruned.count, + removedMutationBytes: mutationBytes ) if !dryRun { + if let cutoff { try await pruneMutations(workspaceId: workspaceId, throughSequence: cutoff) } checkpointsByWorkspace[workspaceId] = retained snapshotsByWorkspace[workspaceId] = retainedSnapshots revisionIndexesByWorkspace[workspaceId] = revisionIndexesByWorkspace[workspaceId]?.filter { @@ -318,7 +366,8 @@ actor InMemoryCheckpointStore: CheckpointStore { /// A JSON file-backed checkpoint store. /// /// Mutations are stored as one JSON line per record in `mutations.jsonl` (append-friendly under -/// the lock). Appends derive the next sequence from the log's final record instead of re-reading +/// the lock). Compaction prunes old records with an ordered atomic replacement. Appends derive +/// the next sequence from the high-water mark and log's final record instead of re-reading /// the whole log, so appends stay cheap as histories grow. A partial trailing line left by a /// crashed append is skipped when reading and truncated before the next append. Writes are /// synchronized through a persistent sidecar lockfile (`mutations.lock`) with an advisory @@ -335,8 +384,16 @@ actor FileCheckpointStore: CheckpointStore { let fileManager: FileManager private let encoder: JSONEncoder private let decoder: JSONDecoder - private let compactEncoder: JSONEncoder + private let compactEncoder = compactStoreEncoder + enum IOEvent: Sendable, Equatable { + case mutationLogSynced + case mutationDecoded + case sequenceProbed + case checkpointWillPublish + case mutationRewriteWillReplace + } + private var ioHook: (@Sendable (IOEvent) throws -> Void)? = nil private let checkpointCapture: Workspace.CheckpointCapturePolicy // Actor reentrancy must suspend contenders before they can conflict with this store's own fd. private var heldLocks: Set = [] @@ -362,10 +419,11 @@ actor FileCheckpointStore: CheckpointStore { encoder.outputFormatting = [.prettyPrinted, .sortedKeys] self.encoder = encoder self.decoder = JSONDecoder() - self.compactEncoder = JSONEncoder() - self.compactEncoder.outputFormatting = [.sortedKeys] } + // Internal fault/ordering hook for persistence-boundary verification. + func setIOHook(_ hook: (@Sendable (IOEvent) throws -> Void)?) { ioHook = hook } + func prepare(workspaceId: UUID) async throws { try await ensureWorkspaceDirectories(for: workspaceId) } @@ -400,7 +458,7 @@ actor FileCheckpointStore: CheckpointStore { parentSnapshot: parentSnapshot ) try writeSnapshotUnlocked(snapshot, workspaceId: draft.workspaceID, reusing: parentManifest?.root) - try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: draft.workspaceID), durability: .publication) + try await publishCheckpoint(checkpoint) listCheckpointsCache[draft.workspaceID] = nil return checkpoint } @@ -627,10 +685,7 @@ actor FileCheckpointStore: CheckpointStore { ) try Task.checkCancellation() try self.write(manifest, to: self.snapshotURL(id: snapshotID, workspaceId: draft.workspaceID), durability: .publication) - try self.write( - checkpoint, - to: self.checkpointURL(id: checkpoint.id, workspaceId: draft.workspaceID), durability: .publication - ) + try await self.publishCheckpoint(checkpoint) self.listCheckpointsCache[draft.workspaceID] = nil return checkpoint } @@ -1090,7 +1145,7 @@ actor FileCheckpointStore: CheckpointStore { if fileManager.fileExists(atPath: jsonl.path) { try appendJSONLLine(encode: record, to: jsonl) } else { - try writeAllMutationsAsJSONL([record], to: jsonl) + try writeAllMutationsAsJSONL([record], to: jsonl, durable: false) } return record } @@ -1104,7 +1159,7 @@ actor FileCheckpointStore: CheckpointStore { let records = fileManager.fileExists(atPath: url.path) ? try loadMutationsFromJSONL(at: url) : [] try state.bumpRetractionEpoch() // Publish invalidation before rewriting. A failed rewrite only causes a harmless reload. - try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .atomic) + try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .publication) try writeAllMutationsAsJSONL(records.filter { $0.sequence != sequence }, to: url) } } @@ -1124,31 +1179,100 @@ actor FileCheckpointStore: CheckpointStore { } } - func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws { + func listMutations(workspaceId: UUID, query: MutationQuery) async throws -> MutationPage { + try query.validate() try await validateWorkspaceFormatIfPresent(workspaceId) - let jsonl = mutationsJsonlURL(workspaceId: workspaceId) - guard fileManager.fileExists(atPath: jsonl.path) else { - return + guard fileManager.fileExists(atPath: workspaceDirectoryURL(workspaceId: workspaceId).path) else { + return MutationPage(records: [], retractionEpoch: 0) } - let lockURL = mutationsLockURL(workspaceId: workspaceId) - try await withExclusiveLock(at: lockURL) { - var state = try mutationSequenceState(workspaceId: workspaceId) - guard fileManager.fileExists(atPath: jsonl.path) else { return } - let records = try loadMutationsFromJSONL(at: jsonl) - .sorted { $0.sequence < $1.sequence } - guard let last = records.last else { - return - } - var kept = records.filter { $0.sequence > throughSequence } - if kept.isEmpty { - kept = [last] + return try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + let state = try readMutationSequenceState(workspaceId: workspaceId) + let url = mutationsJsonlURL(workspaceId: workspaceId) + var records: [Mutation] = [] + if query.limit > 0, fileManager.fileExists(atPath: url.path) { + let start: UInt64 + let end: UInt64? + let reverse: Bool + switch query { + case let .after(cursor, _): + start = try mutationBoundary(at: url, cursor: cursor, inclusive: false) + end = nil + reverse = false + case let .before(cursor, _): + start = 0 + end = try cursor.map { try mutationBoundary(at: url, cursor: $0, inclusive: true) } + reverse = true + } + try scanMutations(at: url, reverse: reverse, startOffset: start, endOffset: end) { record, _ in + records.append(record) + return records.count < query.limit + } } - guard kept.count != records.count else { - return + return MutationPage(records: records, retractionEpoch: state.retractionEpoch) + } + } + + func pruneMutations(workspaceId: UUID, throughSequence: Int) async throws { + _ = try await pruneMutationHistory(workspaceId: workspaceId, throughSequence: throughSequence, dryRun: false) + } + + private func pruneMutationHistory( + workspaceId: UUID, throughSequence: Int?, dryRun: Bool + ) async throws -> MutationPruningResult { + try await validateWorkspaceFormatIfPresent(workspaceId) + let url = mutationsJsonlURL(workspaceId: workspaceId) + guard fileManager.fileExists(atPath: url.path) else { return MutationPruningResult() } + return try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + var state = try readMutationSequenceState(workspaceId: workspaceId) + var result = MutationPruningResult() + let originalBytes = try fileSize(at: url) + var retainedBytes: UInt64 = 0 + // Stage a streamed replacement beside the log. Readers and appends hold the same + // stable lock, so they see either the old log or the complete ordered replacement. + let temporary = url.deletingLastPathComponent().appendingPathComponent(".pending-" + UUID().uuidString + "-mutations.jsonl") + let output: FileHandle? + if dryRun { output = nil } + else { + guard fileManager.createFile(atPath: temporary.path, contents: nil) else { throw POSIXError(.EIO) } + output = try FileHandle(forWritingTo: temporary) + } + defer { try? output?.close(); try? fileManager.removeItem(at: temporary) } + var buffer = Data() + try scanMutations(at: url) { record, line in + state.highWaterMark = max(state.highWaterMark, record.sequence) + if throughSequence.map({ record.sequence <= $0 }) ?? false { + result.count += 1 + } else { + retainedBytes += UInt64(line.count + 1) + if let output { + buffer.append(line) + buffer.append(0x0A) + if buffer.count >= 65536 { + try output.write(contentsOf: buffer) + buffer.removeAll(keepingCapacity: true) + } + } + } + return true } + guard result.count > 0 else { return result } + // Count the actual reduction, including discarded blanks/torn tails and + // normalization of a complete final record that was missing its LF. + result.bytes = originalBytes - retainedBytes + guard !dryRun else { return result } try state.bumpRetractionEpoch() - try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .atomic) - try writeAllMutationsAsJSONL(kept, to: jsonl) + // Invalidation and high-water mark must survive before removing any record. + // An interruption here leaves the old log intact and causes a harmless reload. + try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .publication) + if let output { + try output.write(contentsOf: buffer) + try Self.syncFile(output) + } + try ioHook?(.mutationRewriteWillReplace) + guard rename(temporary.path, url.path) == 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + try Self.syncDirectory(url.deletingLastPathComponent()) + if let output { try Self.syncFile(output) } + return result } } @@ -1182,7 +1306,7 @@ actor FileCheckpointStore: CheckpointStore { ) } return try await withExclusiveLock(at: lifecycleLockURL(workspaceId: workspaceId)) { - try compactUnlocked(workspaceId: workspaceId, retaining: retention, dryRun: dryRun) + try await compactUnlocked(workspaceId: workspaceId, retaining: retention, dryRun: dryRun) } } @@ -1190,7 +1314,7 @@ actor FileCheckpointStore: CheckpointStore { workspaceId: UUID, retaining retention: Workspace.Retention, dryRun: Bool - ) throws -> StoreCompactionResult { + ) async throws -> StoreCompactionResult { let current = try loadAllCheckpointsUnlocked(workspaceId: workspaceId) let before = try physicalStatisticsUnlocked(workspaceId: workspaceId) let plan = try CheckpointRetentionPlanner.plan(checkpoints: current, retaining: retention) @@ -1232,7 +1356,7 @@ actor FileCheckpointStore: CheckpointStore { blobCount: retainedBlobHashes.count + livePacked.count, blobBytes: retainedBlobBytes ) - let report = Workspace.CompactionReport( + var report = Workspace.CompactionReport( dryRun: dryRun, before: before, after: after, @@ -1240,7 +1364,16 @@ actor FileCheckpointStore: CheckpointStore { rebasedCheckpointIDs: plan.rebasedCheckpointIDs ) - guard !dryRun else { return StoreCompactionResult(report: report, checkpoints: retained) } + let cutoff = retained.map(\.mutationCursor).min() + // Validate the entire log before changing checkpoint references or deleting artifacts. + // The real rewrite scans again under the mutations lock to include concurrent changes. + let projectedPruning = try await pruneMutationHistory(workspaceId: workspaceId, + throughSequence: cutoff, dryRun: true) + if dryRun { + report.removedMutationCount = projectedPruning.count + report.removedMutationBytes = projectedPruning.bytes + return StoreCompactionResult(report: report, checkpoints: retained) + } // Rewrite partial packs before changing any references. Then atomically replace // retained snapshot roots; a crash at any point leaves old and/or new packs live. @@ -1261,8 +1394,13 @@ actor FileCheckpointStore: CheckpointStore { trees: &retainedTrees, packs: &retainedPacks, livePacked: &livePacked) } } - for checkpoint in retained where rebasedIDs.contains(checkpoint.id) { - try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: workspaceId), durability: .publication) + let rebased = retained.filter { rebasedIDs.contains($0.id) } + if let cursor = rebased.map(\.mutationCursor).max() { + // Rebasing preserves cursors. One barrier covers every replacement below. + try await syncMutationHistory(workspaceId: workspaceId, throughSequence: cursor) + } + for checkpoint in rebased { + try publishDurableCheckpoint(checkpoint) } for id in plan.removedCheckpointIDs { try removeIfPresent(checkpointURL(id: id, workspaceId: workspaceId)) @@ -1300,16 +1438,28 @@ actor FileCheckpointStore: CheckpointStore { where !retainedBlobHashes.contains(url.lastPathComponent) { try fileManager.removeItem(at: url) } - for directory in [ - workspaceDirectoryURL(workspaceId: workspaceId), checkpointsDirectoryURL(workspaceId: workspaceId), - snapshotsDirectoryURL(workspaceId: workspaceId), blobsDirectoryURL(workspaceId: workspaceId) - ] { - for url in try fileManager.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) - where url.lastPathComponent.hasPrefix(".pending-") { - try fileManager.removeItem(at: url) + // Mutations can still append or retract while the lifecycle lock is held. Do not + // sweep their live replacement files while cleaning up interrupted writes. + try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + for directory in [ + workspaceDirectoryURL(workspaceId: workspaceId), checkpointsDirectoryURL(workspaceId: workspaceId), + snapshotsDirectoryURL(workspaceId: workspaceId), blobsDirectoryURL(workspaceId: workspaceId) + ] { + for url in try fileManager.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + where url.lastPathComponent.hasPrefix(".pending-") { + try fileManager.removeItem(at: url) + } } } try Self.syncDirectory(blobsDirectoryURL(workspaceId: workspaceId)) + // New appends always sequence above every retained cursor, so a validation pass + // with nothing to remove means the rewrite would remove nothing either. + if projectedPruning.count > 0 || projectedPruning.bytes > 0 { + let pruning = try await pruneMutationHistory(workspaceId: workspaceId, + throughSequence: cutoff, dryRun: false) + report.removedMutationCount = pruning.count + report.removedMutationBytes = pruning.bytes + } listCheckpointsCache[workspaceId] = nil return StoreCompactionResult(report: report, checkpoints: retained) } @@ -1504,30 +1654,140 @@ actor FileCheckpointStore: CheckpointStore { } private func loadMutationsFromJSONL(at url: URL, afterSequence: Int = 0) throws -> [Mutation] { - if afterSequence > 0 { - return try loadMutationSuffix(at: url, afterSequence: afterSequence) + var records: [Mutation] = [] + try scanMutations(at: url, reverse: afterSequence > 0) { record, _ in + guard record.sequence > afterSequence else { return false } + records.append(record) + return true } - let data = try Data(contentsOf: url) - if data.isEmpty { return [] } - let endsWithNewline = data.last == UInt8(ascii: "\n") - let lines = data.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false) - var out: [Mutation] = [] - out.reserveCapacity(lines.count) - for (index, line) in lines.enumerated() { - if line.isEmpty { continue } + return afterSequence > 0 ? records.reversed() : records + } + + private struct SequenceProbe: Decodable { let sequence: Int } + + /// Finds the first sequence > cursor (or >= cursor for a reverse page's upper bound). + /// Appends reserve increasing IDs under the mutations lock, retractions sort their + /// replacement, and pruning preserves that order. The caller holds the same lock. + private func mutationBoundary(at url: URL, cursor: Int, inclusive: Bool) throws -> UInt64 { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + let size = try handle.seekToEnd() + func matches(_ sequence: Int) -> Bool { inclusive ? sequence >= cursor : sequence > cursor } + guard let first = try sequenceProbe(in: handle, from: 0, size: size) else { return size } + if matches(first.sequence) { return first.start } + var low = first.end + var high = size + var boundary = size + while low < high { + let middle = low + (high - low) / 2 + // Snap backwards so a very long record cannot make the search skip it. + let start = try Self.lastNewlineOffset(in: handle, before: middle).map { $0 + 1 } ?? 0 + if let probe = try sequenceProbe(in: handle, from: start, size: size) { + if matches(probe.sequence) { + boundary = probe.start + high = start + } else { + low = probe.end + } + } else { + high = start + } + } + return boundary + } + + /// Reads only the sequence, skipping empty lines and an incomplete final append using + /// the same rules as scanMutations. Offsets include a line's LF when one is present. + private func sequenceProbe( + in handle: FileHandle, from offset: UInt64, size: UInt64 + ) throws -> (sequence: Int, start: UInt64, end: UInt64)? { + try handle.seek(toOffset: offset) + var pending = Data() + var start = offset + func decode(_ line: Data, end: UInt64, terminated: Bool) throws -> (sequence: Int, start: UInt64, end: UInt64)? { + guard !line.isEmpty else { return nil } do { - out.append(try decoder.decode(Mutation.self, from: Data(line))) + try ioHook?(.sequenceProbed) + let probe = try decoder.decode(SequenceProbe.self, from: line) + return (probe.sequence, start, end) } catch { - // A crashed append leaves a strict prefix of `\n`: an undecodable final - // line with no trailing newline. Tolerate exactly that torn tail; anything else - // is real corruption and must surface. - if index == lines.count - 1, !endsWithNewline, Self.isIncompleteMutationRecord(Data(line)) { - continue - } + if !terminated, Self.isIncompleteMutationRecord(line) { return nil } + throw error + } + } + while let chunk = try handle.read(upToCount: 65536), !chunk.isEmpty { + pending.append(chunk) + var index = pending.startIndex + while let newline = pending[index...].firstIndex(of: 0x0A) { + let end = start + UInt64(pending.distance(from: index, to: newline)) + 1 + if let probe = try decode(Data(pending[index.. Bool + ) throws { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + let fileSize = try handle.seekToEnd() + let size = endOffset ?? fileSize + guard size > startOffset else { return } + func visit(_ line: Data, final: Bool, terminated: Bool) throws -> Bool { + guard !line.isEmpty else { return true } + let record: Mutation + do { + try ioHook?(.mutationDecoded) + record = try decoder.decode(Mutation.self, from: line) + } + catch { + if final, !terminated, Self.isIncompleteMutationRecord(line) { return true } throw error } + return try consume(record, line) + } + var pending = Data() + if reverse { + try handle.seek(toOffset: size - 1) + let terminated = try handle.read(upToCount: 1) == Data([0x0A]) + var cursor = size + var final = true + while cursor > startOffset { + let start = max(startOffset, cursor > 65536 ? cursor - 65536 : 0) + try handle.seek(toOffset: start) + var chunk = try handle.read(upToCount: Int(cursor - start)) ?? Data() + chunk.append(pending) + let lines = chunk.split(separator: 0x0A, omittingEmptySubsequences: false) + for line in lines.dropFirst().reversed() { + // The empty fragment after a trailing LF is not a record. + if final, line.isEmpty { continue } + if try !visit(Data(line), final: final, terminated: !final || terminated) { return } + final = false + } + pending = Data(lines.first ?? Data.SubSequence()) + cursor = start + } + _ = try visit(pending, final: final, terminated: !final || terminated) + } else { + try handle.seek(toOffset: startOffset) + while let chunk = try handle.read(upToCount: 65536), !chunk.isEmpty { + pending.append(chunk) + var start = pending.startIndex + while let newline = pending[start...].firstIndex(of: 0x0A) { + if try !visit(Data(pending[start.. [Mutation] { - let handle = try FileHandle(forReadingFrom: url) - defer { try? handle.close() } - var cursor = try handle.seekToEnd() - guard cursor > 0 else { return [] } - try handle.seek(toOffset: cursor - 1) - let endsWithNewline = try handle.read(upToCount: 1) == Data([UInt8(ascii: "\n")]) - var pending = Data() - var isFinalRecord = true - var records: [Mutation] = [] - - func consume(_ line: Data) throws -> Bool { - defer { isFinalRecord = false } - guard !line.isEmpty else { return true } - let record: Mutation - do { - record = try decoder.decode(Mutation.self, from: line) - } catch { - if isFinalRecord, !endsWithNewline, Self.isIncompleteMutationRecord(line) { return true } - throw error - } - guard record.sequence > afterSequence else { return false } - records.append(record) - return true - } - - while cursor > 0 { - let start = cursor > 65536 ? cursor - 65536 : 0 - try handle.seek(toOffset: start) - var chunk = try handle.read(upToCount: Int(cursor - start)) ?? Data() - chunk.append(pending) - let lines = chunk.split(separator: UInt8(ascii: "\n"), omittingEmptySubsequences: false) - for line in lines.dropFirst().reversed() { - if try !consume(Data(line)) { return records.reversed() } - } - pending = Data(lines.first ?? Data.SubSequence()) - cursor = start - } - _ = try consume(pending) - return records.reversed() - } - /// Call while holding the lifecycle lock. The cursor is authoritative even when the /// Workspace that requested the checkpoint has not refreshed its mutation cache. func persistedMutationCursor(workspaceId: UUID) async throws -> Int { @@ -1619,6 +1835,38 @@ actor FileCheckpointStore: CheckpointStore { } } + /// All checkpoint publishers hold the lifecycle lock and pass through this barrier. + /// The cursor was frozen before capture; syncing may also include later appends. + func publishCheckpoint(_ checkpoint: Checkpoint) async throws { + try await syncMutationHistory(workspaceId: checkpoint.workspaceID, throughSequence: checkpoint.mutationCursor) + try publishDurableCheckpoint(checkpoint) + } + + private func syncMutationHistory(workspaceId: UUID, throughSequence: Int) async throws { + try await withExclusiveLock(at: mutationsLockURL(workspaceId: workspaceId)) { + var state = try readMutationSequenceState(workspaceId: workspaceId) + state.highWaterMark = max(state.highWaterMark, throughSequence) + let log = mutationsJsonlURL(workspaceId: workspaceId) + if fileManager.fileExists(atPath: log.path) { + // Read the latest valid sequence without repairing the tail. Compaction + // accounts for tail normalization in its own locked replacement. + try scanMutations(at: log, reverse: true) { record, _ in + state.highWaterMark = max(state.highWaterMark, record.sequence) + return false + } + try Self.syncFile(log) + } + try write(state, to: mutationSequenceURL(workspaceId: workspaceId), durability: .publication) + try ioHook?(.mutationLogSynced) + } + } + + /// The caller has synced history through this checkpoint's cursor under the lifecycle lock. + private func publishDurableCheckpoint(_ checkpoint: Checkpoint) throws { + try ioHook?(.checkpointWillPublish) + try write(checkpoint, to: checkpointURL(id: checkpoint.id, workspaceId: checkpoint.workspaceID), durability: .publication) + } + /// Truncates a partial trailing line left behind by a crashed append so subsequent appends /// do not glue new records onto the torn fragment. A torn append never contains a newline /// (it is a strict prefix of `\n`), so everything after the final newline — or the @@ -1691,20 +1939,15 @@ actor FileCheckpointStore: CheckpointStore { return nil } - private func writeAllMutationsAsJSONL(_ records: [Mutation], to url: URL) throws { - if records.isEmpty { - if fileManager.fileExists(atPath: url.path) { - try fileManager.removeItem(at: url) - } - return - } + private func writeAllMutationsAsJSONL(_ records: [Mutation], to url: URL, durable: Bool = true) throws { var data = Data() for record in records.sorted(by: { $0.sequence < $1.sequence }) { var line = try compactEncoder.encode(record) line.append(Data("\n".utf8)) data.append(line) } - try data.write(to: url, options: .atomic) + if durable { try Self.durableWrite(data, to: url, publication: true) } + else { try data.write(to: url, options: .atomic) } } private func appendJSONLLine(encode record: Mutation, to url: URL) throws { @@ -1871,9 +2114,8 @@ actor FileCheckpointStore: CheckpointStore { /// A persistent sidecar lockfile used to serialize mutation-log access. /// - /// The mutations log is append-only; we still take the lock on a **stable** sidecar so the lock - /// is not taken on a file that is unlinked/renamed by atomic write helpers in other - /// subsystems. Mutations are written to `mutations.jsonl`. + /// The lock stays on a stable sidecar while retractions and compaction atomically + /// replace `mutations.jsonl`. private func mutationsLockURL(workspaceId: UUID) -> URL { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("mutations.lock", isDirectory: false) } @@ -2373,6 +2615,23 @@ actor FileCheckpointStore: CheckpointStore { #endif } + private nonisolated static func syncFile(_ url: URL) throws { + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try syncFile(handle) + } + + private nonisolated static func syncFile(_ handle: FileHandle) throws { + try handle.synchronize() +#if canImport(Darwin) + while fcntl(handle.fileDescriptor, F_FULLFSYNC) != 0 { + if errno == EINTR { continue } + // Match durableWrite's fallback for volumes without full-sync support. + break + } +#endif + } + private nonisolated static func syncDirectory(_ url: URL) throws { #if canImport(Darwin) || canImport(Glibc) let fd = open(url.path, O_RDONLY) diff --git a/Sources/Workspace/WorkspaceCompaction.swift b/Sources/Workspace/WorkspaceCompaction.swift index aebbe03..17bcd83 100644 --- a/Sources/Workspace/WorkspaceCompaction.swift +++ b/Sources/Workspace/WorkspaceCompaction.swift @@ -37,6 +37,10 @@ extension Workspace { public var after: StorageStatistics public var removedCheckpointIDs: [UUID] public var rebasedCheckpointIDs: [UUID] + public var removedMutationCount: Int + /// History log bytes reclaimed by removing records. + /// Memory stores report the records' encoded size. + public var removedMutationBytes: UInt64 public var removedCheckpointCount: Int { removedCheckpointIDs.count } public var removedSnapshotCount: Int { max(0, before.snapshotCount - after.snapshotCount) } @@ -48,13 +52,17 @@ extension Workspace { before: StorageStatistics, after: StorageStatistics, removedCheckpointIDs: [UUID], - rebasedCheckpointIDs: [UUID] + rebasedCheckpointIDs: [UUID], + removedMutationCount: Int = 0, + removedMutationBytes: UInt64 = 0 ) { self.dryRun = dryRun self.before = before self.after = after self.removedCheckpointIDs = removedCheckpointIDs self.rebasedCheckpointIDs = rebasedCheckpointIDs + self.removedMutationCount = removedMutationCount + self.removedMutationBytes = removedMutationBytes } } @@ -63,7 +71,9 @@ extension Workspace { try await store.storageStatistics(workspaceId: workspaceId) } - /// Prunes checkpoints, rebases retained lineage, and sweeps unreachable snapshots and blobs. + /// Prunes checkpoints and the history records every retained checkpoint already includes, + /// rebases retained lineage, and sweeps unreachable snapshots and blobs. + /// With no retained checkpoints, mutation history is preserved. /// /// A dry run performs the same validation and returns projected statistics without changing storage. public func compact(retaining retention: Retention, dryRun: Bool = false) async throws -> CompactionReport { diff --git a/Sources/Workspace/WorkspaceHistory.swift b/Sources/Workspace/WorkspaceHistory.swift index 109fda9..03f6fa1 100644 --- a/Sources/Workspace/WorkspaceHistory.swift +++ b/Sources/Workspace/WorkspaceHistory.swift @@ -31,7 +31,7 @@ public struct Mutation: Sendable, Codable, Equatable { } extension Workspace { - /// Returns recorded mutations in ascending sequence order. + /// Returns retained mutation records in ascending sequence order. /// /// History is read from the persistence store, so it includes records appended by other /// workspace instances sharing the store and throws if the stored log can't be read. The call @@ -45,6 +45,25 @@ extension Workspace { return mutations } + /// Returns at most `limit` mutations strictly after `after`, oldest first. + /// A zero limit returns an empty page; a negative limit throws `WorkspaceError.unsupported`. + /// Sequence gaps are preserved. + /// Like `history()`, waits for this instance's mutating operations and reads the store. + public func history(after: Int, limit: Int) async throws -> [Mutation] { + await acquireOperation() + defer { releaseOperation() } + return try await store.listMutations(workspaceId: workspaceId, query: .after(after, limit: limit)).records + } + + /// Returns at most `limit` mutations strictly before `before`, newest first. + /// Pass `nil` explicitly to start at the newest record. A zero limit returns an empty page; + /// a negative limit throws `WorkspaceError.unsupported`. Sequence gaps are preserved. + public func history(before: Int?, limit: Int) async throws -> [Mutation] { + await acquireOperation() + defer { releaseOperation() } + return try await store.listMutations(workspaceId: workspaceId, query: .before(before, limit: limit)).records + } + func refreshPersistedHistory() async throws { // A complete suffix read may cross permanent gaps from retractions or reservations. // An epoch change invalidates the prefix too, including when the log is now empty. diff --git a/Tests/WorkspaceTests/CheckpointStoreTests.swift b/Tests/WorkspaceTests/CheckpointStoreTests.swift index d7b3705..f99bfa2 100644 --- a/Tests/WorkspaceTests/CheckpointStoreTests.swift +++ b/Tests/WorkspaceTests/CheckpointStoreTests.swift @@ -159,7 +159,7 @@ struct CheckpointStoreTests { } try await store.pruneMutations(workspaceId: workspaceID, throughSequence: 100) - #expect(try await store.listMutations(workspaceId: workspaceID).map(\.sequence) == [5]) + #expect(try await store.listMutations(workspaceId: workspaceID).isEmpty) let next = try await store.appendMutation(Self.sampleMutation(workspaceID: workspaceID, path: "/next")) #expect(next.sequence == 6) } diff --git a/Tests/WorkspaceTests/CompactionTests.swift b/Tests/WorkspaceTests/CompactionTests.swift index 7936233..c60e778 100644 --- a/Tests/WorkspaceTests/CompactionTests.swift +++ b/Tests/WorkspaceTests/CompactionTests.swift @@ -36,7 +36,7 @@ struct CompactionTests { #expect(try await workspace.storageStatistics() == applied.after) #expect(try await workspace.checkpoints().map(\.id) == [third.id]) #expect(try await workspace.checkpoint(id: third.id)?.parentID == nil) - #expect(try await workspace.history().count == 3) + #expect(try await workspace.history().isEmpty) await #expect(throws: WorkspaceError.self) { _ = try await workspace.readText("/note", at: .checkpoint(first.id)) } diff --git a/Tests/WorkspaceTests/HistoryDurabilityTests.swift b/Tests/WorkspaceTests/HistoryDurabilityTests.swift new file mode 100644 index 0000000..5805854 --- /dev/null +++ b/Tests/WorkspaceTests/HistoryDurabilityTests.swift @@ -0,0 +1,137 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("History durability") +struct HistoryDurabilityTests { + @Test(arguments: ["manual", "transaction", "restore", "saved snapshot", "compaction rebase"]) + func `checkpoint publishers sync history before exposing metadata`(publisher: String) async throws { + let root = try TestSupport.temporaryDirectory("HistorySyncOrdering") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: .directory(root)) + try await workspace.writeText("/a", "one") + let first = try await workspace.createCheckpoint() + try await workspace.writeText("/a", "two") + _ = try await workspace.createCheckpoint() + let backingStore = await workspace.store + let store = try #require(backingStore as? FileCheckpointStore) + if publisher == "compaction rebase" { + // Two sibling survivors both lose their parent, requiring two publications. + _ = try await store.saveRevision(try await Snapshot.capture(from: workspace.fileSystem), + draft: .init(workspaceID: workspace.workspaceID, preferredParentID: first.id, origin: .manual)) + } + let recorder = StoreIORecorder() + let checkpointDirectory = root.appendingPathComponent(workspace.workspaceID.uuidString).appendingPathComponent("checkpoints") + let baseline = try FileManager.default.contentsOfDirectory(atPath: checkpointDirectory.path).filter { $0.hasSuffix(".json") }.sorted() + await store.setIOHook { event in + recorder.record(event) + if event == .checkpointWillPublish { + let visible = try FileManager.default.contentsOfDirectory(atPath: checkpointDirectory.path).filter { $0.hasSuffix(".json") }.sorted() + #expect(visible == baseline) + #expect(recorder.events.first == .mutationLogSynced) + #expect(recorder.events.dropFirst().allSatisfy { $0 == .checkpointWillPublish }) + } + } + // Appends remain outside the sync barrier. + try await workspace.writeText("/a", "three") + #expect(recorder.events.isEmpty) + switch publisher { + case "manual": _ = try await workspace.createCheckpoint() + case "transaction": + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "committed") + _ = try await transaction.commit() + case "restore": _ = try await workspace.restore(to: first) + case "saved snapshot": + _ = try await store.saveRevision(try await Snapshot.capture(from: workspace.fileSystem), + draft: .init(workspaceID: workspace.workspaceID, preferredParentID: first.id, origin: .manual)) + default: _ = try await workspace.compact(retaining: .latest(2, preservingLabeled: false)) + } + #expect(recorder.events.prefix(2) == [.mutationLogSynced, .checkpointWillPublish]) + #expect(recorder.events.filter { $0 == .mutationLogSynced }.count == 1) + #expect(recorder.events.filter { $0 == .checkpointWillPublish }.count == (publisher == "compaction rebase" ? 2 : 1)) + if publisher != "compaction rebase" { #expect(recorder.events.count == 2) } + } + + @Test(arguments: ["manual", "transaction", "restore"]) + func `a failed history durability barrier prevents publication and compensates mutations`(publisher: String) async throws { + let root = try TestSupport.temporaryDirectory("HistorySyncFailure") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: .directory(root)) + try await workspace.writeText("/a", "one") + let first = try await workspace.createCheckpoint() + try await workspace.writeText("/a", "two") + let before = try await workspace.history() + let backingStore = await workspace.store + let store = try #require(backingStore as? FileCheckpointStore) + let recorder = StoreIORecorder() + await store.setIOHook { event in + recorder.record(event) + if event == .mutationLogSynced { throw TestFileSystemError.injected } + } + await #expect(throws: TestFileSystemError.self) { + switch publisher { + case "manual": _ = try await workspace.createCheckpoint() + case "transaction": + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/a", "committed") + _ = try await transaction.commit() + default: _ = try await workspace.restore(to: first) + } + } + #expect(recorder.events == [.mutationLogSynced]) + #expect(try await workspace.readText("/a") == "two") + let reader = Workspace(workspaceID: workspace.workspaceID, persistence: .directory(root)) + #expect(try await reader.checkpoints().map(\.id) == [first.id]) + #expect(try await reader.history() == before) + } + + @Test + func `interrupted history replacement preserves the old log and permits retry`() async throws { + let root = try TestSupport.temporaryDirectory("InterruptedHistoryRewrite") + defer { TestSupport.remove(root) } + let id = UUID() + let writer = Workspace(workspaceID: id, persistence: .directory(root)) + let reader = Workspace(workspaceID: id, persistence: .directory(root)) + try await writer.writeText("/a", "one") + _ = try await writer.createCheckpoint() + try await writer.writeText("/a", "two") + _ = try await writer.createCheckpoint() + try await writer.writeText("/a", "three") + let before = try await reader.history() + let log = root.appendingPathComponent(id.uuidString).appendingPathComponent("mutations.jsonl") + let bytes = try Data(contentsOf: log) + let backingStore = await writer.store + let store = try #require(backingStore as? FileCheckpointStore) + // Fail after the replacement is synced and the new epoch is durable, before rename. + await store.setIOHook { event in + if event == .mutationRewriteWillReplace { throw TestFileSystemError.injected } + } + await #expect(throws: TestFileSystemError.self) { + _ = try await writer.compact(retaining: .latest(1, preservingLabeled: false)) + } + #expect(try Data(contentsOf: log) == bytes) + #expect(try await reader.history() == before) + #expect(try await reader.history(before: nil, limit: 9) == Array(before.reversed())) + // A restart ignores an abandoned partially written replacement from a killed process. + let abandoned = log.deletingLastPathComponent().appendingPathComponent(".pending-interrupted-mutations.jsonl") + try Data("{partial".utf8).write(to: abandoned) + let restarted = Workspace(workspaceID: id, persistence: .directory(root)) + let applied = try await restarted.compact(retaining: .latest(1, preservingLabeled: false)) + #expect(applied.removedMutationCount == 2) + #expect(try await reader.history().map(\.sequence) == [3]) + #expect(!FileManager.default.fileExists(atPath: abandoned.path)) + try await restarted.writeText("/b", "next") + #expect(try await reader.history().map(\.sequence) == [3, 4]) + } +} + +private final class StoreIORecorder: @unchecked Sendable { + private let lock = NSLock() + private var recorded: [FileCheckpointStore.IOEvent] = [] + var events: [FileCheckpointStore.IOEvent] { lock.withLock { recorded } } + func record(_ event: FileCheckpointStore.IOEvent) { + guard event != .mutationDecoded, event != .sequenceProbed else { return } + lock.withLock { recorded.append(event) } + } +} diff --git a/Tests/WorkspaceTests/HistoryTests.swift b/Tests/WorkspaceTests/HistoryTests.swift new file mode 100644 index 0000000..ff2eb22 --- /dev/null +++ b/Tests/WorkspaceTests/HistoryTests.swift @@ -0,0 +1,349 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Mutation history") +struct HistoryTests { + @Test + func `cursor pages seek with logarithmic probes and decode only the page`() async throws { + let root = try TestSupport.temporaryDirectory("HistorySeek") + defer { TestSupport.remove(root) } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + for index in 1...1000 { + try await store.appendMutation(Mutation(sequence: 0, workspaceID: id, operation: .edit, + changes: .init(changes: [.init(path: WorkspacePath(normalizing: "/file-\(index)"), kind: .file, effect: .created)]))) + } + let counts = HistoryDecodeCounts() + await store.setIOHook { counts.record($0) } + for (query, expected) in [ + (MutationQuery.after(990, limit: 5), Array(991...995)), + (.before(11, limit: 5), Array((6...10).reversed())), + (.before(996, limit: 5), Array((991...995).reversed())) + ] { + counts.reset() + let page = try await store.listMutations(workspaceId: id, query: query) + #expect(page.records.map(\.sequence) == expected) + #expect(counts.full == 5) + #expect(counts.probes <= 24) + } + } + + @Test(arguments: [false, true]) + func `compaction uses the smallest retained cursor even when creation order disagrees`(diskBacked: Bool) async throws { + let root = try TestSupport.temporaryDirectory("HistoryCursorOrder") + defer { TestSupport.remove(root) } + let id = UUID() + let store: any CheckpointStore = diskBacked ? FileCheckpointStore(rootDirectory: root) : InMemoryCheckpointStore() + for _ in 1...3 { + try await store.appendMutation(Mutation(sequence: 0, workspaceID: id, operation: .edit, changes: .init(changes: []))) + } + let files = InMemoryFileSystem() + let firstSnapshot = try await Snapshot.capture(from: files) + let secondSnapshot = try await Snapshot.capture(from: files) + let draft = CheckpointDraft(workspaceID: id, preferredParentID: nil, origin: .manual) + if let memory = store as? InMemoryCheckpointStore { + _ = try await memory.persistRevision(firstSnapshot, draft: draft, mutationCursor: 3) + _ = try await memory.persistRevision(secondSnapshot, draft: draft, mutationCursor: 1) + } else { + _ = try await store.saveRevision(firstSnapshot, draft: draft) + var second = try await store.saveRevision(secondSnapshot, draft: draft) + second.mutationCursor = 1 + let url = root.appendingPathComponent(id.uuidString).appendingPathComponent("checkpoints").appendingPathComponent(second.id.uuidString + ".json") + try JSONEncoder().encode(second).write(to: url) + } + let retained = try await store.listCheckpoints(workspaceId: id) + #expect(retained.map(\.mutationCursor) == [3, 1]) + for dryRun in [true, false] { + let result = try await store.compact(workspaceId: id, retaining: .all, dryRun: dryRun) + #expect(result.report.removedMutationCount == 1) + #expect(try await store.listMutations(workspaceId: id).map(\.sequence) == (dryRun ? [1, 2, 3] : [2, 3])) + } + } + + @Test(arguments: [false, true]) + func `corrupt middle history prevents compaction from deleting any data`(withCheckpoints: Bool) async throws { + let root = try TestSupport.temporaryDirectory("HistoryPreflight") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: .directory(root)) + for index in 1...3 { + try await workspace.writeText("/a", "version \(index)") + if withCheckpoints { _ = try await workspace.createCheckpoint() } + } + let directory = root.appendingPathComponent(workspace.workspaceID.uuidString) + let log = directory.appendingPathComponent("mutations.jsonl") + var lines = try Data(contentsOf: log).split(separator: 0x0A).map { Data($0) } + lines[1] = Data("corrupt middle record".utf8) + var damaged = Data() + for line in lines { damaged.append(line); damaged.append(0x0A) } + try damaged.write(to: log) + func contents() throws -> [String: Data] { + let enumerator = try #require(FileManager.default.enumerator(at: directory, includingPropertiesForKeys: [.isRegularFileKey])) + var result: [String: Data] = [:] + for case let url as URL in enumerator { + if url.pathExtension != "lock", + try url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile == true { + result[url.path] = try Data(contentsOf: url) + } + } + return result + } + let before = try contents() + await #expect(throws: DecodingError.self) { + _ = try await workspace.compact(retaining: .latest(1, preservingLabeled: false)) + } + #expect(try contents() == before) + } + + @Test(arguments: [false, true]) + func `compaction prunes through the oldest retained cursor and reports exact bytes`(diskBacked: Bool) async throws { + let root = try TestSupport.temporaryDirectory("HistoryCompaction") + defer { TestSupport.remove(root) } + let id = UUID() + let writer = Workspace(workspaceID: id, persistence: diskBacked ? .directory(root) : .memory) + try await writer.writeText("/a", "one") + let first = try await writer.createCheckpoint() + try await writer.writeText("/a", "two") + _ = try await writer.createCheckpoint() + try await writer.writeText("/a", "three") + _ = try await writer.createCheckpoint() + try await writer.writeText("/a", "uncheckpointed") + let history = try await writer.history() + let reader = diskBacked ? Workspace(workspaceID: id, persistence: .directory(root)) : writer + #expect(try await reader.history().map(\.sequence) == [1, 2, 3, 4]) + let log = root.appendingPathComponent(id.uuidString).appendingPathComponent("mutations.jsonl") + let original = diskBacked ? try Data(contentsOf: log) : Data() + let preview = try await writer.compact(retaining: .latest(2, preservingLabeled: false), dryRun: true) + #expect(preview.removedMutationCount == 2) + #expect(preview.removedMutationBytes > 0) + #expect(try await writer.history() == history) + if diskBacked { + #expect(try Data(contentsOf: log) == original) + let removedBytes = original.split(separator: 0x0A).prefix(2).reduce(0) { $0 + $1.count + 1 } + #expect(preview.removedMutationBytes == UInt64(removedBytes)) + } + let applied = try await writer.compact(retaining: .latest(2, preservingLabeled: false)) + #expect(applied.removedMutationCount == preview.removedMutationCount) + #expect(applied.removedMutationBytes == preview.removedMutationBytes) + #expect(applied.removedCheckpointIDs == [first.id]) + #expect(try await reader.history().map(\.sequence) == [3, 4]) + #expect(try await reader.history(after: 0, limit: 9).map(\.sequence) == [3, 4]) + #expect(try await reader.history(before: nil, limit: 9).map(\.sequence) == [4, 3]) + #expect(try await reader.history(before: 3, limit: 9).isEmpty) + if diskBacked { #expect(try Data(contentsOf: log).count == original.count - Int(applied.removedMutationBytes)) } + + // Pruning all records must invalidate a populated cache and preserve the high-water mark. + _ = try await writer.createCheckpoint() + _ = try await writer.compact(retaining: .latest(1, preservingLabeled: false)) + #expect(try await reader.history().isEmpty) + let reopened = diskBacked ? Workspace(workspaceID: id, persistence: .directory(root)) : writer + try await reopened.writeText("/b", "next") + #expect(try await reader.history().map(\.sequence) == [5]) + } + + @Test(arguments: [false, true]) + func `compaction with no checkpoints keeps history and all retention uses its oldest checkpoint`(diskBacked: Bool) async throws { + let root = try TestSupport.temporaryDirectory("HistoryWithoutCheckpoints") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: diskBacked ? .directory(root) : .memory) + try await workspace.writeText("/a", "one") + for dryRun in [true, false] { + let report = try await workspace.compact(retaining: .all, dryRun: dryRun) + #expect(report.removedMutationCount == 0) + #expect(report.removedMutationBytes == 0) + #expect(try await workspace.history().map(\.sequence) == [1]) + } + _ = try await workspace.createCheckpoint() + try await workspace.writeText("/a", "two") + _ = try await workspace.createCheckpoint() + let report = try await workspace.compact(retaining: .all) + #expect(report.removedCheckpointCount == 0) + #expect(report.removedMutationCount == 1) + #expect(try await workspace.history().map(\.sequence) == [2]) + } + + @Test(arguments: ["torn tail", "missing newline", "blank lines"]) + func `dry run history bytes match the actual rewrite without repairing the source`(tail: String) async throws { + let root = try TestSupport.temporaryDirectory("HistoryRewriteBytes") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: .directory(root)) + try await workspace.writeText("/a", "checkpointed") + _ = try await workspace.createCheckpoint() + try await workspace.writeText("/a", "checkpointed again") + _ = try await workspace.createCheckpoint() + try await workspace.writeText("/a", "kept") + let directory = root.appendingPathComponent(workspace.workspaceID.uuidString) + let log = directory.appendingPathComponent("mutations.jsonl") + let sidecar = directory.appendingPathComponent("mutations.sequence.json") + var bytes = try Data(contentsOf: log) + switch tail { + case "torn tail": bytes.append(Data([0x7B, 0x22, 0xE2, 0x82])) + case "missing newline": bytes.removeLast() + default: + let lines = bytes.split(separator: 0x0A) + bytes = Data([0x0A, 0x0A]) + for line in lines { bytes.append(line); bytes.append(contentsOf: [0x0A, 0x0A]) } + } + try bytes.write(to: log) + let originalState = try Data(contentsOf: sidecar) + for cursor in [0, 1, 2, 3, 4] { + for limit in [1, 2, 10] { + #expect(try await workspace.history(after: cursor, limit: limit).map(\.sequence) == Array([1, 2, 3].filter { $0 > cursor }.prefix(limit))) + #expect(try await workspace.history(before: cursor, limit: limit).map(\.sequence) == Array([3, 2, 1].filter { $0 < cursor }.prefix(limit))) + } + } + #expect(try await workspace.history(before: nil, limit: 10).map(\.sequence) == [3, 2, 1]) + let projected = try await workspace.compact(retaining: .latest(1, preservingLabeled: false), dryRun: true) + #expect(projected.removedMutationCount == 2) + #expect(try Data(contentsOf: log) == bytes) + #expect(try Data(contentsOf: sidecar) == originalState) + let applied = try await workspace.compact(retaining: .latest(1, preservingLabeled: false)) + #expect(applied.removedMutationCount == 2) + #expect(applied.removedMutationBytes == projected.removedMutationBytes) + #expect(try Data(contentsOf: log).count == bytes.count - Int(applied.removedMutationBytes)) + #expect(try await workspace.history().map(\.sequence) == [3]) + } + + @Test(arguments: [false, true]) + func `exclusive pages preserve ordering gaps and boundaries`(diskBacked: Bool) async throws { + let root = try TestSupport.temporaryDirectory("HistoryPages") + defer { TestSupport.remove(root) } + let workspace = Workspace(persistence: diskBacked ? .directory(root) : .memory) + #expect(try await workspace.history(after: 0, limit: 2).isEmpty) + #expect(try await workspace.history(before: nil, limit: 2).isEmpty) + for index in 1...7 { try await workspace.writeText("/a", "\(index)") } + try await workspace.store.removeMutation(sequence: 3, workspaceId: workspace.workspaceID) + try await workspace.store.removeMutation(sequence: 6, workspaceId: workspace.workspaceID) + let all = try await workspace.history() + #expect(all.map(\.sequence) == [1, 2, 4, 5, 7]) + for cursor in [Int.min, 0, 1, 2, 3, 5, 7, 8, Int.max] { + for limit in [0, 1, 2, 20] { + let ascending = try await workspace.history(after: cursor, limit: limit) + let descending = try await workspace.history(before: cursor, limit: limit) + #expect(ascending == Array(all.filter { $0.sequence > cursor }.prefix(max(0, limit)))) + #expect(descending == Array(all.reversed().filter { $0.sequence < cursor }.prefix(max(0, limit)))) + } + } + let invalidQueries: [MutationQuery] = [.after(3, limit: -1), .before(3, limit: -1), .before(nil, limit: -1)] + for query in invalidQueries { + await #expect { + switch query { + case let .after(cursor, limit): _ = try await workspace.history(after: cursor, limit: limit) + case let .before(cursor, limit): _ = try await workspace.history(before: cursor, limit: limit) + } + } throws: { error in + if case WorkspaceError.unsupported = error { return true } + return false + } + } + #expect(try await workspace.history(before: nil, limit: 3).map(\.sequence) == [7, 5, 4]) + #expect(try await workspace.history(after: 2, limit: 2).map(\.sequence) == [4, 5]) + } + + @Test + func `directory pages stop before decoding unrelated log regions even on a cold workspace`() async throws { + let root = try TestSupport.temporaryDirectory("HistoryStreaming") + defer { TestSupport.remove(root) } + let id = UUID() + let store = FileCheckpointStore(rootDirectory: root) + // Large UTF-8 records cross the scanner's chunk boundaries in both directions. + for index in 1...5 { + let path = WorkspacePath(normalizing: "/\(index)" + String(repeating: "🎉", count: 20000)) + try await store.appendMutation(Mutation(sequence: 0, workspaceID: id, operation: .edit, + changes: .init(changes: [.init(path: path, kind: .file, effect: .created)]))) + } + let workspace = Workspace(workspaceID: id, persistence: .directory(root)) + #expect(try await workspace.history(after: 2, limit: 1).map(\.sequence) == [3]) + #expect(try await workspace.history(before: 4, limit: 1).map(\.sequence) == [3]) + let log = root.appendingPathComponent(id.uuidString).appendingPathComponent("mutations.jsonl") + var lines = try Data(contentsOf: log).split(separator: 0x0A).map { Data($0) } + lines[2] = Data("corrupt middle record".utf8) + var damaged = Data() + for line in lines { damaged.append(line); damaged.append(0x0A) } + try damaged.write(to: log) + #expect(try await workspace.history(after: 0, limit: 2).map(\.sequence) == [1, 2]) + #expect(try await workspace.history(before: nil, limit: 2).map(\.sequence) == [5, 4]) + await #expect(throws: DecodingError.self) { _ = try await workspace.history(after: 0, limit: 3) } + await #expect(throws: DecodingError.self) { _ = try await workspace.history(before: nil, limit: 3) } + await #expect(throws: DecodingError.self) { _ = try await workspace.history() } + } + + @Test + func `pages agree with full history across concurrent appends pruning and retractions`() async throws { + let root = try TestSupport.temporaryDirectory("ConcurrentHistoryPages") + defer { TestSupport.remove(root) } + let id = UUID() + let writer = Workspace(workspaceID: id, persistence: .directory(root)) + let reader = Workspace(workspaceID: id, persistence: .directory(root)) + for index in 1...5 { try await writer.writeText("/a", "\(index)") } + _ = try await writer.createCheckpoint() + _ = try await reader.history() + let append = Task { + for index in 6...30 { try await writer.writeText("/a", "\(index)") } + } + // Use a separate store so pruning and retraction can contend with appends. + let other = FileCheckpointStore(rootDirectory: root) + async let prune = other.compact(workspaceId: id, retaining: .all, dryRun: false) + async let retract: Void = other.removeMutation(sequence: 4, workspaceId: id) + for _ in 0..<20 { + let forward = try await reader.history(after: 5, limit: 3).map(\.sequence) + let backward = try await reader.history(before: nil, limit: 3).map(\.sequence) + #expect(forward == forward.sorted()) + #expect(backward == backward.sorted(by: >)) + #expect(Set(forward).count == forward.count) + #expect(Set(backward).count == backward.count) + } + try await append.value + _ = try await prune + try await retract + var collected: [Mutation] = [] + while true { + let page = try await reader.history(after: collected.last?.sequence ?? 0, limit: 4) + if page.isEmpty { break } + collected += page + } + #expect(collected.map(\.sequence) == Array(6...30)) + #expect(try await reader.history() == collected) + var newest: [Mutation] = [] + while true { + let page = try await reader.history(before: newest.last?.sequence, limit: 4) + if page.isEmpty { break } + newest += page + } + #expect(newest == Array(collected.reversed())) + } + + @Test(arguments: [false, true]) + func `paged history waits for this workspace operation gate`(newestFirst: Bool) async throws { + let files = HookableFileSystem() + let workspace = Workspace(fileSystem: files) + let entered = TestSignal() + let resume = TestSignal() + await files.onWrite { _ in await entered.signal(); await resume.wait() } + let write = Task { try await workspace.writeText("/a", "a") } + await entered.wait() + let read = Task { + if newestFirst { return try await workspace.history(before: nil, limit: 1) } + return try await workspace.history(after: 0, limit: 1) + } + for _ in 0..<100 { await Task.yield() } + await resume.signal() + _ = try await write.value + #expect(try await read.value.map(\.sequence) == [1]) + } +} + +private final class HistoryDecodeCounts: @unchecked Sendable { + private let lock = NSLock() + private var fullCount = 0 + private var probeCount = 0 + var full: Int { lock.withLock { fullCount } } + var probes: Int { lock.withLock { probeCount } } + func reset() { lock.withLock { fullCount = 0; probeCount = 0 } } + func record(_ event: FileCheckpointStore.IOEvent) { + lock.withLock { + if event == .mutationDecoded { fullCount += 1 } + if event == .sequenceProbed { probeCount += 1 } + } + } +} From d7a4e47f7485da0b1bd8b59eacd3528d67459189 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 22:20:36 -0700 Subject: [PATCH 30/34] Create parents in memory, return structured tool errors, drop a dead case InMemoryFileSystem now creates missing parent directories on write, copy, move and symlink creation, matching LocalFileSystem: the same physical resolution (including writes through dangling symlinks and missing components before `..`), permissions from the umask, ENOTDIR for a file in an intermediate position, and quota charging for created parents through Limited. Copying or moving a directory into its own subtree fails with EINVAL on every backend before any parent is created. A parameterized matrix runs each route against Local first and requires identical trees and errors from memory and overlay. Edit capture follows the created parents; it falls back to whole-tree capture only for symlink targets with a `..` after another component. WorkspaceToolAdapter maps failures to WorkspaceToolError, a Codable error with a stable code (not_found, not_a_directory, already_exists, permission_denied, operation_not_permitted, checkpoint_not_found, limit_exceeded, conflict, invalid_arguments, ...), the requested path or scope for path-scoped failures, and a short message a model can act on. Argument errors name the field and the problem; limit errors name the limit; a read that races a write asks for the page to be retried. Messages are fixed strings: host paths, stored spellings, error userInfo and policy deny messages are never forwarded. Checkpoint.Origin.archiveRestore is removed; nothing produced it. --- README.md | 13 +- Sources/Workspace/Checkpoint.swift | 3 +- Sources/Workspace/FS/InMemoryFileSystem.swift | 157 +++++------ Sources/Workspace/FS/LocalFileSystem.swift | 11 +- Sources/Workspace/FS/OverlayFileSystem.swift | 64 +++-- Sources/Workspace/Workspace+Edits.swift | 30 +- Sources/Workspace/WorkspaceToolAdapter.swift | 146 ++++++++-- Sources/Workspace/WorkspaceToolBudget.swift | 4 +- Sources/Workspace/WorkspaceToolError.swift | 154 +++++++++++ Tests/WorkspaceTests/ChangeScalingTests.swift | 48 ++++ Tests/WorkspaceTests/EditPathTests.swift | 16 ++ .../InMemoryTraversalTests.swift | 3 +- Tests/WorkspaceTests/LimitedCacheTests.swift | 5 +- .../WorkspaceTests/ParentCreationTests.swift | 197 ++++++++++++++ .../WorkspaceTests/PathResolutionTests.swift | 6 +- .../WorkspaceToolAdapterTests.swift | 17 +- .../WorkspaceToolErrorTests.swift | 256 ++++++++++++++++++ 17 files changed, 961 insertions(+), 169 deletions(-) create mode 100644 Sources/Workspace/WorkspaceToolError.swift create mode 100644 Tests/WorkspaceTests/ParentCreationTests.swift create mode 100644 Tests/WorkspaceTests/WorkspaceToolErrorTests.swift diff --git a/README.md b/README.md index 8482f7e..317c74a 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,11 @@ targets: [ import Workspace let workspace = Workspace() -try await workspace.createDirectory("/notes") try await workspace.writeText("/notes/todo.txt", "ship it\n") let text = try await workspace.readText("/notes/todo.txt") ``` -`InMemoryFileSystem` requires parent directories to exist before a write; `LocalFileSystem` creates them. +`InMemoryFileSystem` and `LocalFileSystem` create missing parent directories for writes, exclusive file creation, copy and move destinations, and link creation. Created directories have mode `0755` (subject to the host umask on disk). To work on a directory on disk and keep history across launches: @@ -338,6 +337,8 @@ let approvals = PermissionAuthorizer { request in } ``` +`WorkspaceToolAdapter` does not forward deny messages or other policy details to the model. A `PermissionRequest` can carry resolved symlink targets, so those details are deliberately replaced with a generic permission-denied message. + ### Limits - `maxTotalBytes` caps the sum of regular-file sizes, counted per path, so each hard link counts. @@ -381,7 +382,13 @@ Each definition's `inputSchema` is a `JSONValue` holding a plain JSON Schema obj `workspace_apply` finishes the whole edit before trimming its response, so never repeat an apply to recover detail; read the files, or diff from a checkpoint taken first. Diff between checkpoints for stable paging while the tree changes. -`call` throws `WorkspaceToolError` for an unknown tool or invalid arguments. Workspace and filesystem errors, such as a missing file or a denial, propagate unchanged. Turn both into tool-error results for the model. +`call` maps workspace and filesystem failures to `WorkspaceToolError`, a `Codable` value with `code`, optional `path`, and a short `message`. Encode the error as your provider's tool-error result. Codes include `not_found`, `not_a_directory`, `is_a_directory`, `already_exists`, `permission_denied`, `operation_not_permitted`, `invalid_path`, `illegal_byte_sequence`, `limit_exceeded`, `conflict`, `invalid_arguments`, and `unknown_tool`. `checkpoint_not_found` covers missing checkpoints or revision data; `not_found` always means a filesystem entry. Other failures have specific codes such as `symlink_loop`, `directory_not_empty`, `unsupported`, `storage_corrupted`, `rollback_failed`, or the fallback `io_error`. Cancellation remains `CancellationError`. + +An error's `path` is the requested workspace path or scope, using only caller-requested spellings. It is not necessarily the entry that failed: a search root can be reported for a denial deeper within it. Resolved symlink targets, unrequested stored spellings, host paths and underlying error details are omitted. Policy deny messages and other policy details are deliberately dropped because a `PermissionRequest` can carry resolved symlink targets. If the failing path cannot be identified safely, including ambiguous batch or copy/move failures, `path` is absent. Checkpoint, storage, mutation-recording, rollback, transaction and limit errors also omit `path`. For example, reading `/missing.txt` throws an error that encodes as: + +```json +{"code":"not_found","path":"/missing.txt","message":"The requested file or directory was not found."} +``` ## Guarantees and limits diff --git a/Sources/Workspace/Checkpoint.swift b/Sources/Workspace/Checkpoint.swift index cdab063..3c62024 100644 --- a/Sources/Workspace/Checkpoint.swift +++ b/Sources/Workspace/Checkpoint.swift @@ -6,7 +6,6 @@ public struct Checkpoint: Sendable, Codable, Equatable { case manual case rollback(from: UUID) case transaction(base: UUID?) - case archiveRestore } public var id: UUID @@ -27,7 +26,7 @@ public struct Checkpoint: Sendable, Codable, Equatable { switch origin { case let .rollback(from): from case let .transaction(base): base - case .manual, .archiveRestore: nil + case .manual: nil } } diff --git a/Sources/Workspace/FS/InMemoryFileSystem.swift b/Sources/Workspace/FS/InMemoryFileSystem.swift index 99c85f3..59e51f3 100644 --- a/Sources/Workspace/FS/InMemoryFileSystem.swift +++ b/Sources/Workspace/FS/InMemoryFileSystem.swift @@ -233,10 +233,12 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// write a single atomic step. public func createFile(path: WorkspacePath, data: Data) async throws { defer { generation &+= 1 } - if (try? node(at: path, followFinalSymlink: false)) != nil { + let destination = try resolvePath(path: path, followFinalSymlink: false, + allowMissing: true, createParents: true) + if (try? node(at: destination, followFinalSymlink: false)) != nil { throw posixError(EEXIST) } - try write(path: path, data: data, append: false) + try write(path: destination, data: data, append: false) } /// See ``FileSystem/writeFile(path:data:append:)``. @@ -250,7 +252,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E throw posixError(EISDIR) } - let writePath = try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0, allowMissing: true) + let writePath = try resolvePath(path: path, followFinalSymlink: true, allowMissing: true, createParents: true) let (parent, name) = try parentDirectoryAndName(for: writePath) var children = try directoryChildren(of: parent) @@ -346,12 +348,14 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E throw posixError(ENOENT) } + let sourceEntry = try resolvePath(path: sourcePath, followFinalSymlink: false) + let destinationEntry = try resolvePath(path: destinationPath, followFinalSymlink: false, allowMissing: true) if sourceNode.isDirectory, - destinationPath.isWithin(sourcePath) { + destinationEntry.isWithin(sourceEntry) { throw posixError(EINVAL) } - let (destinationParent, destinationName) = try parentDirectoryAndName(for: destinationPath) + let (destinationParent, destinationName) = try parentDirectoryAndName(for: destinationPath, createParents: true) if sourceParent === destinationParent { var children = try directoryChildren(of: sourceParent) if children[PathName(destinationName)] != nil { @@ -385,12 +389,21 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E async throws { defer { generation &+= 1 } - let sourceNode = try node(at: sourcePath, followFinalSymlink: false) + let sourceEntry: WorkspacePath + if sourcePath.isRoot { + sourceEntry = .root + } else { + let parent = try resolvePath(path: sourcePath.dirname, followFinalSymlink: true, requireDirectory: true) + sourceEntry = parent.appending(sourcePath.basename) + } + let destinationEntry = try resolvePath(path: destinationPath, followFinalSymlink: false, allowMissing: true) + guard !destinationEntry.isWithin(sourceEntry) else { throw posixError(EINVAL) } + + let (destinationParent, destinationName) = try parentDirectoryAndName(for: destinationPath, createParents: true) + let sourceNode = try node(at: sourceEntry, followFinalSymlink: false) if sourceNode.isDirectory, !recursive { throw posixError(EISDIR) } - - let (destinationParent, destinationName) = try parentDirectoryAndName(for: destinationPath) var destinationChildren = try directoryChildren(of: destinationParent) if destinationChildren[PathName(destinationName)] != nil { throw posixError(EEXIST) @@ -409,7 +422,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E throw posixError(EEXIST) } - let (parent, name) = try parentDirectoryAndName(for: path) + let (parent, name) = try parentDirectoryAndName(for: path, createParents: true) var children = try directoryChildren(of: parent) if children[PathName(name)] != nil { throw posixError(EEXIST) @@ -432,7 +445,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E throw posixError(EPERM) } - let (parent, name) = try parentDirectoryAndName(for: path) + let (parent, name) = try parentDirectoryAndName(for: path, createParents: true) var children = try directoryChildren(of: parent) if children[PathName(name)] != nil { throw posixError(EEXIST) @@ -465,7 +478,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E /// See ``FileSystem/resolveRealPath(path:)``. public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { - try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0) + try resolvePath(path: path, followFinalSymlink: true) } /// Returns entry existence without following the final symlink (lstat semantics). @@ -574,12 +587,13 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E return children } - private func parentDirectoryAndName(for path: WorkspacePath) throws -> (Node, String) { + private func parentDirectoryAndName(for path: WorkspacePath, createParents: Bool = false) throws -> (Node, String) { guard !path.isRoot else { throw posixError(EPERM) } - let parentPath = path.dirname + let parentPath = try resolvePath(path: path.dirname, followFinalSymlink: true, + allowMissing: createParents, createParents: createParents, requireDirectory: true) let name = path.basename let parent = try node(at: parentPath, followFinalSymlink: true) _ = try directoryChildren(of: parent) @@ -595,93 +609,70 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E return (parent, name, entry) } - private func node(at path: WorkspacePath, followFinalSymlink: Bool, symlinkDepth: Int = 0) throws -> Node { - if symlinkDepth > 64 { - throw posixError(ELOOP) - } - - if path.isRoot { - return root - } - - let components = path.components + private func node(at path: WorkspacePath, followFinalSymlink: Bool) throws -> Node { + let resolved = try resolvePath(path: path, followFinalSymlink: followFinalSymlink) var current = root - var currentPath = WorkspacePath.root - - for (index, component) in components.enumerated() { - guard case let .directory(children) = current.kind else { - throw posixError(ENOTDIR) - } - - guard let child = children[PathName(component)] else { + for component in resolved.components { + guard let child = try directoryChildren(of: current)[PathName(component)] else { throw posixError(ENOENT) } - - let isFinal = index == components.count - 1 - if case let .symlink(target) = child.kind, - (!isFinal || followFinalSymlink) { - let base = currentPath - let targetPath = WorkspacePath(normalizing: target, relativeTo: base) - let remaining = components.suffix(from: index + 1).joined(separator: "/") - let combined = remaining.isEmpty ? targetPath : targetPath.appending(remaining) - return try node(at: combined, followFinalSymlink: followFinalSymlink, symlinkDepth: symlinkDepth + 1) - } - current = child - currentPath = currentPath.appending(component) } - return current } package func resolvePotentialPath(path: WorkspacePath) async throws -> WorkspacePath { - try resolvePath(path: path, followFinalSymlink: true, symlinkDepth: 0, allowMissing: true) + try resolvePath(path: path, followFinalSymlink: true, allowMissing: true) } private func resolvePath( - path: WorkspacePath, followFinalSymlink: Bool, symlinkDepth: Int, allowMissing: Bool = false + path: WorkspacePath, followFinalSymlink: Bool, allowMissing: Bool = false, + createParents: Bool = false, requireDirectory: Bool = false ) throws -> WorkspacePath { - if symlinkDepth > 64 { - throw posixError(ELOOP) - } - - if path.isRoot { - return .root - } - - let components = path.components - var current = root - var resolvedPath = WorkspacePath.root - - for (index, component) in components.enumerated() { - guard case let .directory(children) = current.kind else { - throw posixError(ENOTDIR) + var pending = path.components + var nodes: [Node?] = [root] + var resolved = WorkspacePath.root + var hops = 0 + while !pending.isEmpty { + let component = pending.removeFirst() + if component == "." { continue } + if component == ".." { + guard !resolved.isRoot else { throw WorkspaceError.invalidPath(path.string) } + resolved = resolved.dirname + nodes.removeLast() + continue } - - guard let child = children[PathName(component)] else { - if allowMissing { - return resolvedPath.appending(components.dropFirst(index).joined(separator: "/")) - } + let current = nodes.last! + var children = try current.map { try directoryChildren(of: $0) } ?? [:] + let final = pending.isEmpty + let child: Node? + if let existing = children[PathName(component)] { + child = existing + } else if createParents && (!final || requireDirectory) { + child = makeNode(kind: .directory([:]), permissions: 0o755) + children[PathName(component)] = child + current?.kind = .directory(children) + current?.modificationDate = nextTimestamp() + } else if allowMissing { + child = nil + } else { throw posixError(ENOENT) } - - let isFinal = index == components.count - 1 - if case let .symlink(target) = child.kind, - (!isFinal || followFinalSymlink) { - let targetPath = WorkspacePath(normalizing: target, relativeTo: resolvedPath) - let remaining = components.suffix(from: index + 1).joined(separator: "/") - let combined = remaining.isEmpty ? targetPath : targetPath.appending(remaining) - return try resolvePath( - path: combined, followFinalSymlink: followFinalSymlink, - symlinkDepth: symlinkDepth + 1, allowMissing: allowMissing - ) + if let child, case let .symlink(target) = child.kind, !final || followFinalSymlink { + hops += 1 + guard hops <= 40 else { throw posixError(ELOOP) } + if target.utf8.first == 47 { + nodes = [root] + resolved = .root + } + pending = WorkspacePath.splitComponents(target) + pending + continue } - - current = child - resolvedPath = resolvedPath.appending(component) + if let child, !final || requireDirectory { _ = try directoryChildren(of: child) } + nodes.append(child) + resolved = resolved.appending(component) } - - return resolvedPath + return resolved } private func posixError(_ code: Int32) -> NSError { @@ -690,7 +681,7 @@ public actor InMemoryFileSystem: FileSystem, PotentialPathResolvingFileSystem, E } extension InMemoryFileSystem: PreviewSemanticsFileSystem { - package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { .init() } + package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { .init(createsWriteParents: true) } } /// Optional exact revision for metadata accounting. Each attempted public mutation advances diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index ecd7a96..e922a26 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -943,6 +943,11 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { let config = try requireConfiguration() let source = try resolve(sourcePath, followFinal: false, configuration: config) + let potentialDestination = try resolve(destinationPath, followFinal: false, + allowMissingAncestors: true, creating: true, configuration: config).path + guard potentialDestination == source.path || !potentialDestination.isWithin(source.path) else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EINVAL)) + } let destination = try resolve(destinationPath, followFinal: false, createParents: true, creating: true, configuration: config) let renameError = exclusiveRename(source.parent.value, source.name, destination.parent.value, destination.name) if renameError == 0 { @@ -1044,11 +1049,13 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { let config = try requireConfiguration() let source = try resolve(sourcePath, followFinal: false, configuration: config) - let destination = try resolve(destinationPath, followFinal: false, createParents: true, creating: true, configuration: config) - guard !destination.path.isWithin(source.path) + let potentialDestination = try resolve(destinationPath, followFinal: false, + allowMissingAncestors: true, creating: true, configuration: config).path + guard !potentialDestination.isWithin(source.path) else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EINVAL)) } + let destination = try resolve(destinationPath, followFinal: false, createParents: true, creating: true, configuration: config) try copyEntry(source, to: destination, recursive: recursive) } diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index ea3ae28..5024f92 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -62,7 +62,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { package func previewSemantics(at path: WorkspacePath) async -> FileSystemPreviewSemantics { if let previewSource { return await previewSource.previewBehavior(at: path) } return .init(preservesOverwritePermissions: true, - followsDirectorySymlinks: true, followsPermissionSymlinks: true) + followsDirectorySymlinks: true, followsPermissionSymlinks: true, createsWriteParents: true) } /// Creates and configures an overlay for `rootDirectory`. @@ -247,8 +247,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { throw posixError(EISDIR) } let semantics = await previewSemantics(at: path) - if semantics.createsWriteParents { try await createDirectory(path: path.dirname, recursive: true) } - let target = try await resolveMergedPath(path) + let target = try await resolveMergedPath(path, allowMissingAncestors: semantics.createsWriteParents, createParents: semantics.createsWriteParents) if let info = await upperEntry(target) { try await upper.writeFile(path: target, data: data, append: append) if semantics.preservesOverwritePermissions { try await upper.setPermissions(path: target, permissions: info.permissions) } @@ -258,7 +257,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { guard info.kind != .directory else { throw posixError(EISDIR) } - try await ensureUpperDirectories(for: target) + try await ensureUpperDirectories(for: target, createMissing: semantics.createsWriteParents) let contents: Data if append { contents = try await lower.readFile(path: target) + data @@ -269,17 +268,18 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { try await upper.setPermissions(path: target, permissions: !semantics.preservesOverwritePermissions && !append ? .defaultFile : info.permissions) return } - try await ensureUpperDirectories(for: target) + try await ensureUpperDirectories(for: target, createMissing: semantics.createsWriteParents) try await upper.writeFile(path: target, data: data, append: false) } /// See ``FileSystem/createFile(path:data:)``. public func createFile(path: WorkspacePath, data: Data) async throws { - let path = try await resolveMergedPath(path, followFinal: false) + let createsParents = await previewSemantics(at: path).createsWriteParents + let path = try await resolveMergedPath(path, followFinal: false, allowMissingAncestors: createsParents, createParents: createsParents) guard await mergedEntry(path) == nil else { throw posixError(EEXIST) } - try await ensureUpperDirectories(for: path) + try await ensureUpperDirectories(for: path, createMissing: createsParents) try await upper.createFile(path: path, data: data) } @@ -353,22 +353,25 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { /// See ``FileSystem/move(from:to:)``. public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { let sourcePath = try await resolveMergedPath(sourcePath, followFinal: false) - let destinationPath = try await resolveMergedPath(destinationPath, followFinal: false) - if sourcePath == destinationPath { + let createsParents = await previewSemantics(at: destinationPath).createsWriteParents + let potentialDestination = try await resolveMergedPath(destinationPath, followFinal: false, allowMissingAncestors: createsParents) + if sourcePath == potentialDestination { return } guard let sourceInfo = await mergedEntry(sourcePath) else { throw posixError(ENOENT) } - if sourceInfo.kind == .directory, destinationPath.isWithin(sourcePath) { + if sourceInfo.kind == .directory, potentialDestination.isWithin(sourcePath) { throw posixError(EINVAL) } + let destinationPath = try await resolveMergedPath(destinationPath, followFinal: false, + allowMissingAncestors: createsParents, createParents: createsParents) guard await mergedEntry(destinationPath) == nil else { throw posixError(EEXIST) } try await materializeTree(sourcePath) - try await ensureUpperDirectories(for: destinationPath) + try await ensureUpperDirectories(for: destinationPath, createMissing: createsParents) try await upper.move(from: sourcePath, to: destinationPath) if await lowerVisibleEntry(sourcePath) != nil { cuts = cuts.filter { !$0.isWithin(sourcePath) } @@ -381,36 +384,41 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { async throws { let sourcePath = try await resolveMergedPath(sourcePath, followFinal: false) - let destinationPath = try await resolveMergedPath(destinationPath, followFinal: false) + let createsParents = await previewSemantics(at: destinationPath).createsWriteParents + let potentialDestination = try await resolveMergedPath(destinationPath, followFinal: false, allowMissingAncestors: createsParents) + guard !potentialDestination.isWithin(sourcePath) else { throw posixError(EINVAL) } guard let sourceInfo = await mergedEntry(sourcePath) else { throw posixError(ENOENT) } if sourceInfo.kind == .directory, !recursive { throw posixError(EISDIR) } + let destinationPath = try await resolveMergedPath(destinationPath, followFinal: false, + allowMissingAncestors: createsParents, createParents: createsParents) guard await mergedEntry(destinationPath) == nil else { throw posixError(EEXIST) } try await materializeTree(sourcePath) - try await ensureUpperDirectories(for: destinationPath) + try await ensureUpperDirectories(for: destinationPath, createMissing: createsParents) try await upper.copy(from: sourcePath, to: destinationPath, recursive: recursive) } /// See ``FileSystem/createSymlink(path:target:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { - let path = try await resolveMergedPath(path, followFinal: false) _ = try WorkspacePath(validating: target, relativeTo: path.dirname) + let createsParents = await previewSemantics(at: path).createsWriteParents + let path = try await resolveMergedPath(path, followFinal: false, allowMissingAncestors: createsParents, createParents: createsParents) guard await mergedEntry(path) == nil else { throw posixError(EEXIST) } - try await ensureUpperDirectories(for: path) + try await ensureUpperDirectories(for: path, createMissing: createsParents) try await upper.createSymlink(path: path, target: target) } /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { - let path = try await resolveMergedPath(path, followFinal: false) + let createsParents = await previewSemantics(at: path).createsWriteParents let target = try await resolveMergedPath(target) guard let targetInfo = await mergedEntry(target) else { throw posixError(ENOENT) @@ -418,6 +426,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { guard targetInfo.kind != .directory else { throw posixError(EPERM) } + let path = try await resolveMergedPath(path, followFinal: false, allowMissingAncestors: createsParents, createParents: createsParents) guard await mergedEntry(path) == nil else { throw posixError(EEXIST) } @@ -425,7 +434,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { if await upperEntry(target) == nil { try await copyUpLeaf(target, info: targetInfo) } - try await ensureUpperDirectories(for: path) + try await ensureUpperDirectories(for: path, createMissing: createsParents) try await upper.createHardLink(path: path, target: target) } @@ -477,7 +486,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { /// Resolve each component in the merged namespace before consulting a backing filesystem. /// A lower-layer ancestor link must never let that backend bypass cuts or upper entries. private func resolveMergedPath( - _ path: WorkspacePath, followFinal: Bool = true, allowMissingAncestors: Bool = false + _ path: WorkspacePath, followFinal: Bool = true, allowMissingAncestors: Bool = false, createParents: Bool = false ) async throws -> WorkspacePath { var current = WorkspacePath.root var pending = path.components @@ -488,12 +497,19 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { continue } if component == ".." { + guard !current.isRoot else { throw WorkspaceError.invalidPath(path.string) } current = current.dirname continue } let next = current.appending(component) let final = pending.isEmpty guard let info = await mergedEntry(next) else { + if !final && createParents { + try await ensureUpperDirectories(for: next) + try await upper.createDirectory(path: next, recursive: false) + current = next + continue + } if final || allowMissingAncestors { current = next continue @@ -528,17 +544,19 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { } /// Materializes every merged-visible ancestor directory of `path` into the upper layer, - /// preserving lower-layer permissions. Throws `ENOENT` when an ancestor does not exist in - /// the merged view (parents are not invented, matching the in-memory backend). - private func ensureUpperDirectories(for path: WorkspacePath) async throws { + /// preserving lower-layer permissions. Creation operations may also create missing parents. + private func ensureUpperDirectories(for path: WorkspacePath, createMissing: Bool = false) async throws { var current = WorkspacePath.root for component in path.dirname.components { current = current.appending(component) - if await upperEntry(current) != nil { + if let info = await upperEntry(current) { + guard info.kind == .directory else { throw posixError(ENOTDIR) } continue } guard let info = await lowerVisibleEntry(current) else { - throw posixError(ENOENT) + guard createMissing else { throw posixError(ENOENT) } + try await upper.createDirectory(path: current, recursive: false) + continue } guard info.kind == .directory else { throw posixError(ENOTDIR) diff --git a/Sources/Workspace/Workspace+Edits.swift b/Sources/Workspace/Workspace+Edits.swift index d3aae41..83a7ff0 100644 --- a/Sources/Workspace/Workspace+Edits.swift +++ b/Sources/Workspace/Workspace+Edits.swift @@ -82,14 +82,20 @@ extension Workspace { var roots: [WorkspacePath] = [] let resolvesPaths = await filesystem.capabilities().contains(.realPathResolution) for path in paths { - let entryPath = try await canonicalEntryPath(path, resolvesPaths: resolvesPaths) + let entryPath: WorkspacePath + if resolvesPaths && !path.isRoot { + let parent = try await resolvedCapturePath(path.dirname, additionalRoots: &roots) + entryPath = parent.appending(path.basename) + } else { + entryPath = path + } roots.append(try await captureRoot(for: entryPath)) // Some backends replace the leaf link, while others follow it. Capture // both the canonical entry and its target, but never lexical aliases. switch edit { case .writeText, .appendText, .writeData, .appendData, .setPermissions, .replace: if resolvesPaths { - let target = try await resolvedCapturePath(entryPath) + let target = try await resolvedCapturePath(entryPath, additionalRoots: &roots) roots.append(try await captureRoot(for: target)) } default: @@ -101,34 +107,44 @@ extension Workspace { func canonicalEntryPath(_ path: WorkspacePath, resolvesPaths: Bool) async throws -> WorkspacePath { guard resolvesPaths, !path.isRoot else { return path } - let parent = try await resolvedCapturePath(path.dirname) + // Archive restore rebases every write to this destination, so it never traverses the original link. + var additionalRoots: [WorkspacePath] = [] + let parent = try await resolvedCapturePath(path.dirname, additionalRoots: &additionalRoots) return parent.appending(path.basename) } // Resolve existing ancestors, preserving missing suffixes for creation edits. // A dangling leaf still has a target even when resolveRealPath requires that // target to exist. Only absence permits fallback; authorization errors escape. - private func resolvedCapturePath(_ path: WorkspacePath, linkDepth: Int = 0) async throws -> WorkspacePath { + private func resolvedCapturePath(_ path: WorkspacePath, additionalRoots: inout [WorkspacePath], linkDepth: Int = 0) async throws -> WorkspacePath { guard linkDepth < 64 else { throw WorkspaceError.invalidPath("symbolic link cycle at \(path)") } if path.isRoot { return try await filesystem.resolveRealPath(path: path) } - let parent = try await resolvedCapturePath(path.dirname, linkDepth: linkDepth) + let parent = try await resolvedCapturePath(path.dirname, additionalRoots: &additionalRoots, linkDepth: linkDepth) let entry = parent.appending(path.basename) guard let info = try await Snapshot.infoIfPresent(at: entry, on: filesystem) else { return entry } guard info.kind == .symlink else { return entry } do { let resolved = try await filesystem.resolveRealPath(path: entry) if resolved != entry { - return try await resolvedCapturePath(resolved, linkDepth: linkDepth + 1) + return try await resolvedCapturePath(resolved, additionalRoots: &additionalRoots, linkDepth: linkDepth + 1) } } catch { guard Snapshot.isMissingEntry(error) else { throw error } } let target = try await filesystem.readSymlink(path: entry) + let components = WorkspacePath.splitComponents(target).filter { $0 != "." } + if components.drop(while: { $0 == ".." }).contains("..") { + // Leading '..' is exact from the physically resolved parent. A later + // '..' may leave a newly created directory or a symlink, outside the final + // canonical destination. Capture the tree conservatively in this rare + // case; the backend owns absolute-target namespaces and traversal. + additionalRoots.append(.root) + } // Absolute fallback targets are interpreted as workspace paths. let targetPath = WorkspacePath(normalizing: target, relativeTo: entry.dirname) - return try await resolvedCapturePath(targetPath, linkDepth: linkDepth + 1) + return try await resolvedCapturePath(targetPath, additionalRoots: &additionalRoots, linkDepth: linkDepth + 1) } func captureRoot(for path: WorkspacePath) async throws -> WorkspacePath { diff --git a/Sources/Workspace/WorkspaceToolAdapter.swift b/Sources/Workspace/WorkspaceToolAdapter.swift index c32091a..7a84039 100644 --- a/Sources/Workspace/WorkspaceToolAdapter.swift +++ b/Sources/Workspace/WorkspaceToolAdapter.swift @@ -45,18 +45,6 @@ public struct WorkspaceToolDefinition: Sendable, Codable, Equatable { } } -public enum WorkspaceToolError: Error, Sendable, Equatable, CustomStringConvertible { - case unknownTool(String) - case invalidArguments(String) - - public var description: String { - switch self { - case let .unknownTool(name): "unknown workspace tool: \(name)" - case let .invalidArguments(message): "invalid workspace tool arguments: \(message)" - } - } -} - /// Dependency-free tool definitions and execution over one shared Workspace authority. public actor WorkspaceToolAdapter { public let workspace: Workspace @@ -145,7 +133,7 @@ public actor WorkspaceToolAdapter { "edits": .arraySchema( items: .schema( properties: [ - "kind": .enumSchema(["write_text", "append_text", "create_directory", "remove", "copy", "move"]), + "kind": .enumSchema(ToolEdit.Kind.allCases.map(\.rawValue)), "path": .stringSchema("Target workspace path."), "content": .stringSchema("Text for write_text or append_text."), "source": .stringSchema("Source path for copy or move."), @@ -186,10 +174,12 @@ public actor WorkspaceToolAdapter { let decoder = JSONDecoder() let encoder = JSONEncoder() encoder.outputFormatting = [.sortedKeys] + var requestedPaths: [WorkspacePath] = [] do { switch name { case "workspace_read": let request = try decoder.decode(ReadArguments.self, from: arguments) + requestedPaths = [request.path] let maxBytes = try boundedToolInteger(request.maxBytes, default: Self.defaultReadMaxBytes, minimum: Self.minimumReadMaxBytes, maximum: Self.maximumReadMaxBytes, name: "max_bytes") let offset = request.offset ?? 0 @@ -208,7 +198,8 @@ public actor WorkspaceToolAdapter { hasMore: offset < totalBytes && UInt64(data.count) < totalBytes - offset, offset: offset, leadingContext: leadingContext) guard end > 0 || offset >= totalBytes else { - throw WorkspaceError.unsupported("file changed or returned an incomplete read at \(request.path); retry the page") + throw WorkspaceToolError(code: .conflict, path: request.path, + message: "The file changed during the read; retry this page.") } // The read length was bounded by totalBytes - offset, so this cannot overflow. let nextOffset = offset + UInt64(end) @@ -216,6 +207,7 @@ public actor WorkspaceToolAdapter { nextOffset: nextOffset, totalBytes: totalBytes, truncated: nextOffset < totalBytes)) case "workspace_glob": let request = try decoder.decode(GlobArguments.self, from: arguments) + requestedPaths = [request.currentDirectory ?? .root] let budget = try toolResultBudget(request.maxBytes) let offset = try boundedToolInteger(request.matchOffset, default: 0, minimum: 0, maximum: Int.max, name: "match_offset") let count = try boundedToolInteger(request.maxMatches, default: Self.defaultGlobMaxMatches, minimum: 1, @@ -225,13 +217,24 @@ public actor WorkspaceToolAdapter { return try boundedGlobOutput(matches, offset: offset, maxMatches: count, maxBytes: budget, encoder: encoder) case "workspace_search": let request = try decoder.decode(SearchArguments.self, from: arguments) + requestedPaths = [request.root] return try encoder.encode(try await workspace.search(request.request())) case "workspace_apply": let request = try decoder.decode(ApplyArguments.self, from: arguments) + requestedPaths = request.edits.flatMap { [$0.path] + ($0.source.map { [$0] } ?? []) } let budget = try toolResultBudget(request.maxBytes) let fileBudget = try toolFileBudget(request.maxFileBytes) - let result = try await workspace.apply(try request.edits.map { try $0.edit }) - return boundedApplyOutput(result.changes, maxBytes: budget, maxFileBytes: fileBudget, encoder: encoder) + do { + let result = try await workspace.apply(try request.edits.map { try $0.edit }) + return boundedApplyOutput(result.changes, maxBytes: budget, maxFileBytes: fileBudget, encoder: encoder) + } catch { + if await isSubtreeTransferFailure(error, edits: request.edits) { + let failure = WorkspaceToolError(mapping: error, requestedPaths: requestedPaths) + throw WorkspaceToolError(code: failure.code, path: failure.path, + message: "The destination is inside the source.") + } + throw error + } case "workspace_diff": let request = try decoder.decode(DiffArguments.self, from: arguments) let budget = try toolResultBudget(request.maxBytes) @@ -259,7 +262,30 @@ public actor WorkspaceToolAdapter { } catch let error as WorkspaceToolError { throw error } catch let error as DecodingError { - throw WorkspaceToolError.invalidArguments(String(describing: error)) + throw toolArgumentError(error) + } catch is CancellationError { + throw CancellationError() + } catch { + throw WorkspaceToolError(mapping: error, requestedPaths: requestedPaths) + } + } + + private func isSubtreeTransferFailure(_ error: any Error, edits: [ToolEdit]) async -> Bool { + let failure = error as NSError + guard failure.domain == NSPOSIXErrorDomain, failure.code == Int(POSIXError.Code.EINVAL.rawValue), + edits.count == 1, let edit = edits.first, edit.kind == .copy || edit.kind == .move, + let source = edit.source else { return false } + // A batch can fail on several operations, and EINVAL has other causes. + // Only specialize an unambiguous transfer with confirmed containment. + let fs = workspace.fileSystem + let resolver = FileSystemPathResolver(base: fs) + let features = await fs.capabilities() + do { + let from = try await resolver.resolve(source, features: features, followFinal: false) + let to = try await resolver.resolve(edit.path, features: features, followFinal: false) + return to.isWithin(from) + } catch { + return false } } } @@ -335,6 +361,11 @@ private struct SearchArguments: Decodable { maxBytes = try container.decodeIfPresent(Int.self, forKey: .maxBytes) } func request() throws -> SearchRequest { + guard !pattern.isEmpty else { throw WorkspaceToolError.invalidArguments("The pattern must not be empty.") } + if regularExpression { + do { _ = try NSRegularExpression(pattern: pattern) } + catch { throw WorkspaceToolError.invalidArguments("The pattern must be a valid regular expression.") } + } let context = try boundedToolInteger(contextLines, default: 0, minimum: 0, maximum: SearchRequest.maximumContextLines, name: "context_lines") let offset = try boundedToolInteger(matchOffset, default: 0, minimum: 0, maximum: Int.max, name: "match_offset") @@ -364,8 +395,25 @@ private struct ApplyArguments: Decodable { } } -private struct ToolEdit: Decodable { - var kind: String +private struct ToolEdit: Decodable, Sendable { + enum Kind: String, Decodable, CaseIterable, Sendable { + case writeText = "write_text" + case appendText = "append_text" + case createDirectory = "create_directory" + case remove, copy, move + + init(from decoder: any Decoder) throws { + let raw = try decoder.singleValueContainer().decode(String.self) + guard let kind = Self(rawValue: raw) else { + let valid = Self.allCases.map(\.rawValue).joined(separator: ", ") + throw WorkspaceToolError.invalidArguments( + "Unsupported value at \(toolCodingPath(decoder.codingPath)); use one of \(valid).") + } + self = kind + } + } + + var kind: Kind var path: WorkspacePath var content: String? var source: WorkspacePath? @@ -374,13 +422,12 @@ private struct ToolEdit: Decodable { var edit: Edit { get throws { switch kind { - case "write_text": .writeText(path, try required(content, "content")) - case "append_text": .appendText(path, try required(content, "content")) - case "create_directory": .createDirectory(path, recursive: recursive ?? true) - case "remove": .remove(path, recursive: recursive ?? true) - case "copy": .copy(from: try required(source, "source"), to: path, recursive: recursive ?? true) - case "move": .move(from: try required(source, "source"), to: path) - default: throw WorkspaceToolError.invalidArguments("unsupported edit kind \(kind)") + case .writeText: .writeText(path, try required(content, "content")) + case .appendText: .appendText(path, try required(content, "content")) + case .createDirectory: .createDirectory(path, recursive: recursive ?? true) + case .remove: .remove(path, recursive: recursive ?? true) + case .copy: .copy(from: try required(source, "source"), to: path, recursive: recursive ?? true) + case .move: .move(from: try required(source, "source"), to: path) } } } @@ -413,13 +460,52 @@ private struct CheckpointArguments: Decodable { var label: String? } private func toolRevision(_ value: String?) throws -> Revision { guard let value else { return .current } guard let id = UUID(uuidString: value) else { - throw WorkspaceToolError.invalidArguments("checkpoint id is not a UUID: \(value)") + throw WorkspaceToolError.invalidArguments("The checkpoint ID must be a UUID.") } return .checkpoint(id) } +// Coding paths contain schema field names and array indexes, never argument values. +// Do not use DecodingError.debugDescription or underlying errors in model output. +private func toolArgumentError(_ error: DecodingError) -> WorkspaceToolError { + let keys: [any CodingKey] + let problem: String + switch error { + case let .keyNotFound(key, context): + keys = context.codingPath + [key] + problem = "Missing required value" + case let .valueNotFound(_, context): + keys = context.codingPath + problem = "Expected a non-null value" + case let .typeMismatch(type, context): + keys = context.codingPath + if type == String.self { problem = "Expected a string" } + else if type == Bool.self { problem = "Expected a boolean" } + else if type == Int.self || type == UInt64.self { problem = "Expected an integer" } + else if type == Double.self { problem = "Expected a number" } + else if type == [Any].self { problem = "Expected an array" } + else if type == [String: Any].self { problem = "Expected an object" } + else { problem = "Expected a value of the required type" } + case let .dataCorrupted(context): + keys = context.codingPath + problem = "Invalid value or malformed JSON" + @unknown default: + keys = [] + problem = "Invalid arguments" + } + return .invalidArguments("\(problem) at \(toolCodingPath(keys)).") +} + +private func toolCodingPath(_ keys: [any CodingKey]) -> String { + let path = keys.reduce(into: "") { result, key in + if let index = key.intValue { result += "[\(index)]" } + else { result += (result.isEmpty ? "" : ".") + key.stringValue } + } + return path.isEmpty ? "arguments" : path +} + private func required(_ value: T?, _ name: String) throws -> T { - guard let value else { throw WorkspaceToolError.invalidArguments("missing \(name)") } + guard let value else { throw WorkspaceToolError.invalidArguments("Missing required value at \(name).") } return value } @@ -457,7 +543,7 @@ private extension JSONValue { private func boundedToolInteger(_ value: Int?, default fallback: Int, minimum: Int, maximum: Int, name: String) throws -> Int { let value = value ?? fallback guard value >= minimum else { - throw WorkspaceToolError.invalidArguments("\(name) must be at least \(minimum)") + throw WorkspaceToolError.invalidArguments("The \(name) argument must be at least \(minimum).") } return min(value, maximum) } @@ -476,7 +562,7 @@ private func toolTextPage( // Report invalidEncoding instead if the rest of the page also contains invalid bytes. _ = try toolTextPage(Data(data.dropFirst(leading.count)), path: path, hasMore: hasMore) throw WorkspaceToolError.invalidArguments( - "offset \(offset) is inside a UTF-8 character; use a nextOffset value from a previous read" + "The offset is inside a UTF-8 character; use a nextOffset value from a previous read." ) } throw WorkspaceError.invalidEncoding(path) diff --git a/Sources/Workspace/WorkspaceToolBudget.swift b/Sources/Workspace/WorkspaceToolBudget.swift index 75e1177..44ef6ff 100644 --- a/Sources/Workspace/WorkspaceToolBudget.swift +++ b/Sources/Workspace/WorkspaceToolBudget.swift @@ -114,7 +114,7 @@ func boundedChangesOutput( if patch { output.unifiedPatch = patchSections.joined(separator: "\n") } let data = try encoder.encode(output) guard data.count <= maxBytes else { - throw WorkspaceToolError.invalidArguments("max_bytes cannot fit result metadata") + throw WorkspaceToolError.invalidArguments("The max_bytes argument cannot fit the result metadata.") } return data } @@ -151,7 +151,7 @@ func boundedGlobOutput( output.truncated = cursor < matches.count || output.omittedMatches > 0 let data = try encoder.encode(output) guard data.count <= maxBytes else { - throw WorkspaceToolError.invalidArguments("max_bytes cannot fit result metadata") + throw WorkspaceToolError.invalidArguments("The max_bytes argument cannot fit the result metadata.") } return data } diff --git a/Sources/Workspace/WorkspaceToolError.swift b/Sources/Workspace/WorkspaceToolError.swift new file mode 100644 index 0000000..ef567f4 --- /dev/null +++ b/Sources/Workspace/WorkspaceToolError.swift @@ -0,0 +1,154 @@ +import Foundation + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// A model-facing tool failure. Encode this value as the provider's tool-error result. +/// Messages never embed underlying error descriptions or resolved filesystem names. +public struct WorkspaceToolError: Error, Sendable, Codable, Equatable, CustomStringConvertible { + public enum Code: String, Sendable, Codable { + case unknownTool = "unknown_tool" + case invalidArguments = "invalid_arguments" + case notFound = "not_found" + case checkpointNotFound = "checkpoint_not_found" + case notADirectory = "not_a_directory" + case isADirectory = "is_a_directory" + case alreadyExists = "already_exists" + case permissionDenied = "permission_denied" + case operationNotPermitted = "operation_not_permitted" + case invalidPath = "invalid_path" + case illegalByteSequence = "illegal_byte_sequence" + case limitExceeded = "limit_exceeded" + case conflict + case directoryNotEmpty = "directory_not_empty" + case symlinkLoop = "symlink_loop" + case unsupported + case storageCorrupted = "storage_corrupted" + case rollbackFailed = "rollback_failed" + case ioError = "io_error" + } + + public let code: Code + /// A caller-requested workspace path or scope, which need not name the failing entry. + /// Resolved targets and stored alias spellings are never substituted here. + public let path: WorkspacePath? + public let message: String + + public init(code: Code, path: WorkspacePath? = nil, message: String) { + self.code = code + self.path = path + self.message = message + } + + public var description: String { + "\(code.rawValue)\(path.map { " at \($0)" } ?? ""): \(message)" + } + + public static func unknownTool(_ name: String) -> Self { + let scalars = name.unicodeScalars.prefix(129) + let prefix = String(String.UnicodeScalarView(scalars.prefix(128))) + let displayed = prefix + (scalars.count > 128 ? "…" : "") + return Self(code: .unknownTool, path: nil, message: "Unknown workspace tool: \(displayed).") + } + + public static func invalidArguments(_ message: String) -> Self { + Self(code: .invalidArguments, path: nil, message: message) + } + + init(mapping error: any Error, requestedPaths: [WorkspacePath]) { + // Never substitute a backend-reported spelling. Even a valid virtual path + // may disclose a resolved target the caller did not request. + let candidates = Set(requestedPaths) + var path = candidates.count == 1 ? candidates.first : nil + var pathScoped = false + let code: Code + let message: String + switch error { + case is WorkspaceRollbackError: + code = .rollbackFailed; message = "The operation failed and the previous state could not be fully restored. Inspect the workspace before retrying." + case let limit as FileSystemLimitError: + code = .limitExceeded + switch limit { + case let .writeBytes(_, configured): + message = "The per-write byte limit is \(configured) bytes. Split the write into smaller calls." + case let .totalBytes(_, configured): + message = "The total byte limit is \(configured) bytes. Free space before retrying." + case let .entryCount(_, configured): + message = "The entry count limit is \(configured). Remove entries before retrying." + } + case let workspace as WorkspaceError: + switch workspace { + case .invalidPath: + pathScoped = true + code = .invalidPath; message = "The path is invalid or resolves outside the workspace." + case .readOnly, .accessDenied: + pathScoped = true + code = .permissionDenied; message = "Workspace access is denied for this operation." + case let .invalidEncoding(reported): + pathScoped = true + if candidates.contains(reported) { path = reported } + code = .illegalByteSequence; message = "The file is not valid UTF-8 text." + case .decodingFailed: + code = .invalidArguments; message = "The file does not contain valid JSON for this operation." + case .unsupported: + code = .unsupported; message = "This operation is not supported with these arguments or on this filesystem." + case .checkpointNotFound, .revisionDataNotFound: + code = .checkpointNotFound; message = "The requested checkpoint or revision data was not found." + case .mutationFailed: + code = .ioError; message = "The workspace mutation could not be recorded." + case .storageCorrupted: + code = .storageCorrupted; message = "Workspace revision storage is damaged." + } + case is Workspace.Transaction.TransactionError: + code = .conflict; message = "The transaction conflicts with the current workspace state." + default: + let posix = error as NSError + if posix.domain == NSPOSIXErrorDomain { + pathScoped = true + switch posix.code { + case Int(ENOENT): code = .notFound; message = "The requested file or directory was not found." + case Int(ENOTDIR): code = .notADirectory; message = "A path component is not a directory." + case Int(EISDIR): code = .isADirectory; message = "The path is a directory; this operation requires a file." + case Int(EEXIST): code = .alreadyExists; message = "An entry already exists at the destination." + case Int(EPERM): code = .operationNotPermitted; message = "The filesystem does not permit this operation on this path." + case Int(EACCES), Int(EROFS): code = .permissionDenied; message = "Workspace access is denied for this operation." + case Int(EINVAL), Int(ENAMETOOLONG): code = .invalidPath; message = "The path is invalid for this operation." + case Int(EILSEQ): code = .illegalByteSequence; message = "The path or content contains an invalid byte sequence." + case Int(EFBIG), Int(ENOSPC), Int(EDQUOT): pathScoped = false; code = .limitExceeded; message = "The operation exceeds a filesystem size or capacity limit." + case Int(EBUSY), Int(EAGAIN): code = .conflict; message = "The operation conflicts with the current filesystem state; retry after refreshing." + case Int(ENOTEMPTY): code = .directoryNotEmpty; message = "The directory is not empty." + case Int(ELOOP): code = .symlinkLoop; message = "The path contains too many symbolic links or a symbolic link cycle." + case Int(ENOTSUP): code = .unsupported; message = "The filesystem does not support this operation." + default: code = .ioError; message = "The workspace filesystem operation failed." + } + } else if posix.domain == NSCocoaErrorDomain { + pathScoped = CocoaError(_nsError: posix).isFileError + switch posix.code { + case NSFileNoSuchFileError, NSFileReadNoSuchFileError: + code = .notFound; message = "The requested file or directory was not found." + case NSFileReadNoPermissionError, NSFileWriteNoPermissionError, NSFileWriteVolumeReadOnlyError: + code = .permissionDenied; message = "Workspace access is denied for this operation." + case NSFileWriteFileExistsError: + code = .alreadyExists; message = "An entry already exists at the destination." + case NSFileReadInvalidFileNameError, NSFileWriteInvalidFileNameError: + code = .invalidPath; message = "The path is invalid for this operation." + case NSFileReadInapplicableStringEncodingError: + code = .illegalByteSequence; message = "The file is not valid text in the requested encoding." + case NSFileWriteOutOfSpaceError: + pathScoped = false + code = .limitExceeded; message = "The filesystem has insufficient capacity for this operation." + default: + code = .ioError; message = "The workspace filesystem operation failed." + } + } else { + // Backend and persistence errors may contain host paths, policy details, + // or stored spellings. Never forward their description or userInfo. + code = .ioError; message = "The workspace operation failed." + } + } + self.init(code: code, path: pathScoped ? path : nil, message: message) + } +} diff --git a/Tests/WorkspaceTests/ChangeScalingTests.swift b/Tests/WorkspaceTests/ChangeScalingTests.swift index 35d1a71..2427f74 100644 --- a/Tests/WorkspaceTests/ChangeScalingTests.swift +++ b/Tests/WorkspaceTests/ChangeScalingTests.swift @@ -58,6 +58,24 @@ struct ChangeScalingTests { #expect(await counted.listings.isEmpty) } + @Test(arguments: ["../missing.txt", "./.././missing.txt", "../shared/.env"], [false, true]) + func `leading dotdot dangling targets do not read unrelated files`(target: String, localBackend: Bool) async throws { + let root = try TestSupport.temporaryDirectory("LeadingDotdotJournal") + defer { TestSupport.remove(root) } + let base: any FileSystem = localBackend ? try LocalFileSystem(root: root) : InMemoryFileSystem() + try await base.createDirectory(path: "/a", recursive: true) + try await base.writeFile(path: "/other/file", data: Data("unrelated".utf8), append: false) + try await base.createSymlink(path: "/a/link", target: target) + let reads = JournalReadCounter(base: base) + let workspace = Workspace(fileSystem: reads) + let changes = try await workspace.writeText("/a/link", "new") + let destination = try WorkspacePath(target, relativeTo: "/a") + #expect(changes.touchedPaths.contains(destination)) + #expect(try await base.readFile(path: destination) == Data("new".utf8)) + #expect(await reads.paths.contains(destination)) + #expect(!(await reads.paths.contains("/other/file"))) + } + @Test func `preview excludes untouched symlinks and reads only affected contents`() async throws { let root = try TestSupport.temporaryDirectory() @@ -261,3 +279,33 @@ final class SignallingFileSystem: FileSystemChangeSource, ExactTimestampFileSyst func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) } } + +// Forward real-path resolution so this counter exercises the journal's canonical-path route. +private actor JournalReadCounter: FileSystem { + let base: any FileSystem + var paths: Set = [] + init(base: any FileSystem) { self.base = base } + func stat(path: WorkspacePath) async throws -> FileInfo { try await base.stat(path: path) } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func readFile(path: WorkspacePath) async throws -> Data { + paths.insert(path) + return try await base.readFile(path: path) + } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await base.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { + try await base.writeFile(path: path, data: data, append: append) + } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await base.remove(path: path, recursive: recursive) } + func move(from: WorkspacePath, to: WorkspacePath) async throws { try await base.move(from: from, to: to) } + func copy(from: WorkspacePath, to: WorkspacePath, recursive: Bool) async throws { try await base.copy(from: from, to: to, recursive: recursive) } + func createSymlink(path: WorkspacePath, target: String) async throws { try await base.createSymlink(path: path, target: target) } + func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { try await base.createHardLink(path: path, target: target) } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } + func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { try await base.setPermissions(path: path, permissions: permissions) } + func capabilities() async -> FileSystemFeatures { await base.capabilities() } + func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { try await base.resolveRealPath(path: path) } +} diff --git a/Tests/WorkspaceTests/EditPathTests.swift b/Tests/WorkspaceTests/EditPathTests.swift index 295a515..bdd92d4 100644 --- a/Tests/WorkspaceTests/EditPathTests.swift +++ b/Tests/WorkspaceTests/EditPathTests.swift @@ -55,6 +55,21 @@ struct EditPathTests { static let scenarios: [Scenario] = [ .init(description: "nested mount write", edits: [.writeText("/mnt/data/x", "hi")]), .init(description: "overwrite mode", edits: [.setPermissions("/dir/a", POSIXPermissions(0o600)), .writeText("/dir/a", "new")]), + .init(description: "missing component before dotdot", edits: [ + .createSymbolicLink("/dotdot", target: "scratch/../dir"), + .writeText("/dotdot/new/file", "new"), + ]), + .init(description: "missing writeText", edits: [.writeText("/new/deep/file", "new")]), + .init(description: "missing appendText", edits: [.appendText("/new/deep/file", "new")]), + .init(description: "missing writeData", edits: [.writeData("/new/deep/file", Data([1, 2]))]), + .init(description: "missing appendData", edits: [.appendData("/new/deep/file", Data([1, 2]))]), + .init(description: "missing copy", edits: [.copy(from: "/dir/sub", to: "/new/deep/copied")]), + .init(description: "missing move", edits: [.move(from: "/dir/sub", to: "/new/deep/moved")]), + .init(description: "missing symbolic link", edits: [.createSymbolicLink("/new/deep/link", target: "../../dir/a")]), + .init(description: "missing hard link", edits: [.createHardLink("/new/deep/hard", target: "/dir/a")]), + .init(description: "missing alias parents", edits: [.writeText("/dirlink/new/deep/file", "new")]), + .init(description: "dangling parent write", edits: [.writeText("/dangling/deep/file", "new")]), + .init(description: "dangling leaf parents", edits: [.writeText("/deepdangling", "new")]), .init(description: "writeText", edits: [.writeText("/dir/a", "new")]), .init(description: "appendText", edits: [.appendText("/dir/a", "new")]), .init(description: "writeData", edits: [.writeData("/dir/a", Data([1, 2]))]), @@ -128,6 +143,7 @@ struct EditPathTests { try await fs.createSymlink(path: "/link", target: "dir/a") try await fs.createSymlink(path: "/dangling", target: "nowhere") try await fs.createSymlink(path: "/chain", target: "dangling") + try await fs.createSymlink(path: "/deepdangling", target: "new/deep/file") let workspace = Workspace(fileSystem: fs) let before = try await Snapshot.capture(from: fs) let edits = scenario.edits + (failing ? [.copy(from: "/absent", to: "/failure")] : []) diff --git a/Tests/WorkspaceTests/InMemoryTraversalTests.swift b/Tests/WorkspaceTests/InMemoryTraversalTests.swift index b785522..dab8916 100644 --- a/Tests/WorkspaceTests/InMemoryTraversalTests.swift +++ b/Tests/WorkspaceTests/InMemoryTraversalTests.swift @@ -7,6 +7,7 @@ struct InMemoryTraversalTests { func nativeWalkMatchesPerPathCaptureAndMetadata(skipUnreadable: Bool) async throws { let fs = InMemoryFileSystem() try await fs.createDirectory(path: "/a/empty", recursive: true) + try await fs.writeFile(path: "/implicit/parents/file", data: Data([1, 2]), append: false) for path: WorkspacePath in ["/a/file", "/a.txt", "/a0", "/é", "/e\u{301}"] { try await fs.writeFile(path: path, data: Data([0, 1, 255]), append: false) } @@ -52,7 +53,7 @@ struct InMemoryTraversalTests { #expect(nativeBytes == fallbackBytes) #expect(nativeLinks == fallbackLinks) if path.isRoot { - #expect(nativeInfos.filter { $0.kind == .file }.map(\.path) == ["/a.txt", "/a/file", "/a0", "/e\u{301}", "/hard", "/é"]) + #expect(nativeInfos.filter { $0.kind == .file }.map(\.path) == ["/a.txt", "/a/file", "/a0", "/e\u{301}", "/hard", "/implicit/parents/file", "/é"]) #expect(nativeInfos.first { $0.path == "/hard" }?.linkCount == 2) #expect(nativeInfos.first { $0.path == "/hard" }?.identity == nativeInfos.first { $0.path == "/a/file" }?.identity) } diff --git a/Tests/WorkspaceTests/LimitedCacheTests.swift b/Tests/WorkspaceTests/LimitedCacheTests.swift index d1f6436..268c2fc 100644 --- a/Tests/WorkspaceTests/LimitedCacheTests.swift +++ b/Tests/WorkspaceTests/LimitedCacheTests.swift @@ -267,15 +267,14 @@ struct LimitedCacheTests { let memory = InMemoryFileSystem() let fs = LimitedFileSystem(base: memory, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) try await checkTotals(fs, base: memory) - try await memory.writeFile(path: "/external", data: Data(count: 5), append: false) + try await memory.writeFile(path: "/new/deep/external", data: Data(count: 5), append: false) try await checkTotals(fs, base: memory) - try await memory.createDirectory(path: "/dir", recursive: false) try await memory.createFile(path: "/dir/file", data: Data(count: 2)) try await memory.createHardLink(path: "/alias", target: "/dir/file") try await checkTotals(fs, base: memory) try await memory.writeFile(path: "/alias", data: Data(count: 1), append: true) try await checkTotals(fs, base: memory) - try await memory.move(from: "/external", to: "/moved") + try await memory.move(from: "/new/deep/external", to: "/moved") try await memory.copy(from: "/dir", to: "/copy", recursive: true) try await memory.createSymlink(path: "/link", target: "/absent") try await checkTotals(fs, base: memory) diff --git a/Tests/WorkspaceTests/ParentCreationTests.swift b/Tests/WorkspaceTests/ParentCreationTests.swift new file mode 100644 index 0000000..8c5456a --- /dev/null +++ b/Tests/WorkspaceTests/ParentCreationTests.swift @@ -0,0 +1,197 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Implicit parent creation") +struct ParentCreationTests { + enum Operation: String, CaseIterable, Sendable { + case replace, append, create, copy, move, symlink, hardLink + + func perform(on fs: any FileSystem, at path: WorkspacePath, source: WorkspacePath = "/source") async throws { + switch self { + case .replace, .append: try await fs.writeFile(path: path, data: Data("new".utf8), append: self == .append) + case .create: try await fs.createFile(path: path, data: Data("new".utf8)) + case .copy: try await fs.copy(from: source, to: path, recursive: true) + case .move: try await fs.move(from: source, to: path) + case .symlink: try await fs.createSymlink(path: path, target: "../../source") + case .hardLink: try await fs.createHardLink(path: path, target: "/source") + } + } + } + + static let cases = Operation.allCases.flatMap { operation in + ["plain", "alias", "dangling", "leaf", "file", "file alias", "absolute", "missing before dotdot", "escape", "loop", "subtree", "subtree source alias", "subtree destination alias", "missing source subtree"] + .filter { $0 != "leaf" || [.replace, .append, .create].contains(operation) } + .filter { $0 != "missing source subtree" || operation == .copy } + .filter { !$0.hasPrefix("subtree") || [.copy, .move].contains(operation) } + .map { (operation, $0) } + } + + // Host creation modes depend on umask. Compare memory defaults after applying the + // modes observed from Local, retaining explicit permissions and all tree structure. + private func comparable(_ entry: Snapshot.Entry, diskRoot: String? = nil, + directoryMode: POSIXPermissions = .defaultDirectory, + fileMode: POSIXPermissions = .defaultFile) -> Snapshot.Entry { + switch entry { + case var .directory(directory): + if !directory.path.isRoot && diskRoot == nil { + directory.permissions = POSIXPermissions(Int(directory.permissions.rawValue & directoryMode.rawValue)) + } + directory.children = directory.children.map { + comparable($0, diskRoot: diskRoot, directoryMode: directoryMode, fileMode: fileMode) + } + return .directory(directory) + case var .symlink(link): + link.permissions = POSIXPermissions(0o777) + if let diskRoot, link.target.hasPrefix(diskRoot + "/") { + link.target = String(link.target.dropFirst(diskRoot.count)) + } + return .symlink(link) + case var .file(file): + if diskRoot == nil && file.path != "/source" && file.path != "/d/f" { + file.permissions = POSIXPermissions(Int(file.permissions.rawValue & fileMode.rawValue)) + } + return .file(file) + default: return entry + } + } + + @Test(arguments: cases) + func `creation matches disk including parents permissions and errors`(operation: Operation, route: String) async throws { + let root = try TestSupport.temporaryDirectory("ParentCreation") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.createDirectory(path: "/mode-probe", recursive: false) + try await local.createFile(path: "/mode-probe/file", data: Data()) + let directoryMode = try await local.stat(path: "/mode-probe").permissions + let fileMode = try await local.stat(path: "/mode-probe/file").permissions + try await local.remove(path: "/mode-probe", recursive: true) + let diskRoot = root.resolvingSymlinksInPath().path + let memory = InMemoryFileSystem() + let lower = InMemoryFileSystem() + let overlay = OverlayFileSystem(over: lower) + let backends: [any FileSystem] = [local, memory, overlay] + for fs in [local as any FileSystem, memory, lower] { + try await fs.setPermissions(path: "/", permissions: .defaultDirectory) + try await fs.createDirectory(path: "/existing", recursive: true) + try await fs.setPermissions(path: "/existing", permissions: POSIXPermissions(0o700)) + try await fs.writeFile(path: "/source", data: Data("source".utf8), append: false) + try await fs.setPermissions(path: "/source", permissions: .defaultFile) + try await fs.createSymlink(path: "/alias", target: "existing") + try await fs.createSymlink(path: "/dangling", target: "new") + try await fs.createSymlink(path: "/leaf", target: "new/deep/file") + try await fs.createSymlink(path: "/escape", target: "../outside") + try await fs.createSymlink(path: "/loop", target: "loop") + try await fs.createSymlink(path: "/file-alias", target: "source") + try await fs.createSymlink(path: "/absolute", target: fs is LocalFileSystem ? diskRoot + "/existing" : "/existing") + try await fs.createSymlink(path: "/dotdot", target: "scratch/../existing") + try await fs.createFile(path: "/d/f", data: Data("tree".utf8)) + try await fs.setPermissions(path: "/d/f", permissions: .defaultFile) + try await fs.createSymlink(path: "/d-alias", target: "d") + try await fs.createSymlink(path: "/source-parent", target: ".") + } + let lowerBefore = try await Snapshot.capture(from: lower) + let path: WorkspacePath = switch route { + case "alias": "/alias/new/deep/file" + case "dangling": "/dangling/deep/file" + case "leaf": "/leaf" + case "file": "/source/deep/file" + case "file alias": "/file-alias/deep/file" + case "absolute": "/absolute/new/deep/file" + case "missing before dotdot": "/dotdot/new/deep/file" + case "subtree", "subtree source alias": "/d/new/x" + case "subtree destination alias": "/d-alias/new/x" + case "missing source subtree": "/missing/x" + case "escape": "/escape/deep/file" + case "loop": "/loop/deep/file" + default: "/new/deep/file" + } + var expectedTree: Snapshot.Entry? + var expectedError: String? + for fs in backends { + var failure: String? + let before = try await Snapshot.capture(from: fs).entry + let source: WorkspacePath = route == "missing source subtree" ? "/missing" : route == "subtree source alias" ? "/source-parent/d" : (route.hasPrefix("subtree") ? "/d" : "/source") + do { try await operation.perform(on: fs, at: path, source: source) } + catch WorkspaceError.invalidPath { failure = "invalid_path" } + catch { failure = "\((error as NSError).domain):\((error as NSError).code)" } + let after = try await Snapshot.capture(from: fs).entry + if route.hasPrefix("subtree") || route == "missing source subtree" { + #expect(failure == "\(NSPOSIXErrorDomain):\(EINVAL)") + #expect(after == before) + } + let tree = comparable(after, diskRoot: fs is LocalFileSystem ? diskRoot : nil, + directoryMode: directoryMode, + fileMode: [.copy, .move, .hardLink].contains(operation) ? .defaultFile : fileMode) + if let expectedTree { + #expect(tree == expectedTree) + #expect(failure == expectedError) + } else { + expectedTree = tree + expectedError = failure + switch route { + case "file", "file alias": #expect(failure == "\(NSPOSIXErrorDomain):\(ENOTDIR)") + case "escape": #expect(failure == "invalid_path") + case "subtree", "subtree source alias", "subtree destination alias", "missing source subtree": + #expect(failure == "\(NSPOSIXErrorDomain):\(EINVAL)") + case "loop": #expect(failure == "\(NSPOSIXErrorDomain):\(ELOOP)") + case "leaf" where operation != .replace && operation != .append: + #expect(failure == "\(NSPOSIXErrorDomain):\(EEXIST)") + default: #expect(failure == nil) + } + } + } + #expect(try await Snapshot.capture(from: lower).entry == lowerBefore.entry) + } + + @Test(arguments: [false, true]) + func `transaction draft creates parents and commits the same tree`(localBackend: Bool) async throws { + let root = try TestSupport.temporaryDirectory("DraftParents") + defer { TestSupport.remove(root) } + let fs: any FileSystem = localBackend ? try LocalFileSystem(root: root) : InMemoryFileSystem() + let workspace = Workspace(fileSystem: fs) + let before = try await Snapshot.capture(from: fs) + let transaction = try await workspace.beginTransaction() + try await transaction.writeText("/new/deep/file", "draft") + #expect(try await Snapshot.capture(from: fs).entry == before.entry) + let commit = try await transaction.commit() + #expect(commit.applied) + #expect(try await workspace.readText("/new/deep/file") == "draft") + let after = try await Snapshot.capture(from: fs) + #expect(commit.appliedChanges == ChangeSet.compare(before: before, after: after, maxTextBytes: DiffOptions.default.maxTextBytes)) + } + + @Test(arguments: Operation.allCases, ["plain", "dangling", "alias"]) + func `quota charges all implicit parents and retains correct accounting`(operation: Operation, route: String) async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/source", data: Data("source".utf8), append: false) + let path: WorkspacePath + switch route { + case "dangling": + try await memory.createSymlink(path: "/link", target: "new") + path = "/link/deep/file" + case "alias": + try await memory.createDirectory(path: "/existing", recursive: true) + try await memory.createSymlink(path: "/link", target: "existing") + path = "/link/new/deep/file" + default: path = "/new/deep/file" + } + let before = try await Snapshot.capture(from: memory) + let total = (operation == .move ? 3 : 4) + (route == "alias" ? 2 : (route == "dangling" ? 1 : 0)) + let limited = LimitedFileSystem(base: memory, limits: .init(maxEntryCount: total - 1)) + await #expect(throws: FileSystemLimitError.entryCount(attempted: total, limit: total - 1)) { + try await operation.perform(on: limited, at: path) + } + #expect(try await Snapshot.capture(from: memory).entry == before.entry) + let enough = LimitedFileSystem(base: memory, limits: .init(maxEntryCount: total)) + try await operation.perform(on: enough, at: path) + // The retained accounting must also charge those entries on the next mutation. + await #expect(throws: FileSystemLimitError.entryCount(attempted: total + 1, limit: total)) { + try await enough.createFile(path: "/extra", data: Data()) + } + try await enough.revalidate() + await #expect(throws: FileSystemLimitError.entryCount(attempted: total + 1, limit: total)) { + try await enough.createFile(path: "/extra", data: Data()) + } + } +} diff --git a/Tests/WorkspaceTests/PathResolutionTests.swift b/Tests/WorkspaceTests/PathResolutionTests.swift index 486068f..300b501 100644 --- a/Tests/WorkspaceTests/PathResolutionTests.swift +++ b/Tests/WorkspaceTests/PathResolutionTests.swift @@ -195,9 +195,9 @@ struct PathResolutionTests { await #expect(throws: (any Error).self) { _ = try await fs.stat(path: "/alias/file") } - await #expect(throws: (any Error).self) { - try await fs.createFile(path: "/alias/new", data: Data()) - } + try await fs.createFile(path: "/alias/new", data: Data()) + #expect(try await fs.listDirectory(path: "/dir").map(\.name) == ["new"]) + #expect(try await base.readFile(path: "/dir/file") == Data("lower".utf8)) } @Test diff --git a/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift b/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift index d1dbd1a..24714cd 100644 --- a/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift +++ b/Tests/WorkspaceTests/WorkspaceToolAdapterTests.swift @@ -203,7 +203,7 @@ struct WorkspaceToolAdapterTests { try await workspace.writeData(Data([0x61, 0x80, 0x62]), to: "/orphan") let adapter = WorkspaceToolAdapter(workspace: workspace) await #expect(throws: WorkspaceToolError.invalidArguments( - "offset 2 is inside a UTF-8 character; use a nextOffset value from a previous read" + "The offset is inside a UTF-8 character; use a nextOffset value from a previous read." )) { _ = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/unicode","offset":2,"max_bytes":4}"#.utf8)) } @@ -214,12 +214,9 @@ struct WorkspaceToolAdapterTests { do { _ = try await adapter.call(name: "workspace_read", arguments: Data(arguments.utf8)) Issue.record("expected invalidEncoding for malformed bytes") - } catch let error as WorkspaceError { - guard case let .invalidEncoding(invalidPath) = error else { - Issue.record("expected invalidEncoding, received \(error)") - continue - } - #expect(invalidPath == path) + } catch let error as WorkspaceToolError { + #expect(error.code == .illegalByteSequence) + #expect(error.path == path) } } let first = try JSONDecoder().decode(ReadPage.self, from: await adapter.call( @@ -254,9 +251,9 @@ struct WorkspaceToolAdapterTests { arguments: Data("{\"path\":\"/missing\"\(suffix)}".utf8)) Issue.record("expected ENOENT for missing file") } catch { - let posix = error as NSError - #expect(posix.domain == NSPOSIXErrorDomain) - #expect(posix.code == Int(ENOENT)) + let failure = try #require(error as? WorkspaceToolError) + #expect(failure.code == .notFound) + #expect(failure.path == "/missing") } } } diff --git a/Tests/WorkspaceTests/WorkspaceToolErrorTests.swift b/Tests/WorkspaceTests/WorkspaceToolErrorTests.swift new file mode 100644 index 0000000..d749afa --- /dev/null +++ b/Tests/WorkspaceTests/WorkspaceToolErrorTests.swift @@ -0,0 +1,256 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Model-facing tool errors") +struct WorkspaceToolErrorTests { + struct Failure: Sendable { + var error: any Error + var code: WorkspaceToolError.Code + var pathScoped = false + var message: String? + } + + static let failures: [Failure] = { + let posix: [(Int32, WorkspaceToolError.Code)] = [ + (ENOENT, .notFound), (ENOTDIR, .notADirectory), (EISDIR, .isADirectory), + (EEXIST, .alreadyExists), (EACCES, .permissionDenied), (EPERM, .operationNotPermitted), + (EROFS, .permissionDenied), (EINVAL, .invalidPath), (ENAMETOOLONG, .invalidPath), + (EILSEQ, .illegalByteSequence), (EFBIG, .limitExceeded), (ENOSPC, .limitExceeded), + (EDQUOT, .limitExceeded), (EBUSY, .conflict), (EAGAIN, .conflict), + (ENOTEMPTY, .directoryNotEmpty), (ELOOP, .symlinkLoop), (ENOTSUP, .unsupported), (EIO, .ioError), + ] + let cocoa: [(Int, WorkspaceToolError.Code)] = [ + (NSFileNoSuchFileError, .notFound), (NSFileReadNoSuchFileError, .notFound), + (NSFileReadNoPermissionError, .permissionDenied), (NSFileWriteNoPermissionError, .permissionDenied), + (NSFileWriteVolumeReadOnlyError, .permissionDenied), (NSFileWriteFileExistsError, .alreadyExists), + (NSFileReadInvalidFileNameError, .invalidPath), (NSFileWriteInvalidFileNameError, .invalidPath), + (NSFileReadInapplicableStringEncodingError, .illegalByteSequence), + (NSFileWriteOutOfSpaceError, .limitExceeded), (NSFileReadUnknownError, .ioError), + ] + // Underlying messages, userInfo, and even typed paths may contain private names. + let privateDetails = "/private/host/Stored-Spelling" + let info = [NSFilePathErrorKey: privateDetails, NSLocalizedDescriptionKey: privateDetails] + let filesystem = posix.map { Failure(error: NSError(domain: NSPOSIXErrorDomain, code: Int($0.0), userInfo: info), code: $0.1, pathScoped: $0.1 != .limitExceeded) } + + cocoa.map { Failure(error: NSError(domain: NSCocoaErrorDomain, code: $0.0, userInfo: info), code: $0.1, pathScoped: $0.1 != .limitExceeded) } + let workspace: [(WorkspaceError, WorkspaceToolError.Code, Bool)] = [ + (.invalidPath(privateDetails), .invalidPath, true), (.readOnly, .permissionDenied, true), + (.accessDenied(operation: privateDetails, message: privateDetails), .permissionDenied, true), + (.invalidEncoding("/Stored-Spelling"), .illegalByteSequence, true), + (.decodingFailed("/Stored-Spelling", underlying: privateDetails), .invalidArguments, false), + (.unsupported(privateDetails), .unsupported, false), (.checkpointNotFound(UUID()), .checkpointNotFound, false), + (.revisionDataNotFound(UUID()), .checkpointNotFound, false), (.mutationFailed(privateDetails), .ioError, false), + (.storageCorrupted(privateDetails), .storageCorrupted, false), + ] + return filesystem + workspace.map { Failure(error: $0.0, code: $0.1, pathScoped: $0.2) } + [ + .init(error: FileSystemLimitError.totalBytes(attempted: 200, limit: 123), code: .limitExceeded, + message: "The total byte limit is 123 bytes. Free space before retrying."), + .init(error: FileSystemLimitError.entryCount(attempted: 200, limit: 123), code: .limitExceeded, + message: "The entry count limit is 123. Remove entries before retrying."), + .init(error: FileSystemLimitError.writeBytes(attempted: 200, limit: 123), code: .limitExceeded, + message: "The per-write byte limit is 123 bytes. Split the write into smaller calls."), + .init(error: Workspace.Transaction.TransactionError.conflicts([]), code: .conflict), + .init(error: Workspace.Transaction.TransactionError.committing, code: .conflict), + .init(error: Workspace.Transaction.TransactionError.inactive, code: .conflict), + .init(error: WorkspaceRollbackError(originalError: WorkspaceError.invalidPath(privateDetails), + rollbackError: WorkspaceError.storageCorrupted(privateDetails)), code: .rollbackFailed), + .init(error: NSError(domain: privateDetails, code: 1, userInfo: info), code: .ioError), + .init(error: NSError(domain: NSCocoaErrorDomain, code: NSPropertyListReadCorruptError, userInfo: info), code: .ioError), + ] + }() + + @Test(arguments: failures) + func `call maps errors without disclosing backend details`(failure: Failure) async throws { + let fs = HookableFileSystem() + try await fs.writeFile(path: "/requested", data: Data([1]), append: false) + await fs.onRead { _ in throw failure.error } + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: fs)) + do { + _ = try await adapter.call(name: "workspace_apply", arguments: Data(#"{"edits":[{"kind":"write_text","path":"/requested","content":"new"}]}"#.utf8)) + Issue.record("expected tool error") + } catch let error as WorkspaceToolError { + #expect(error.code == failure.code) + let expectedPath: WorkspacePath? = failure.pathScoped ? "/requested" : nil + #expect(error.path == expectedPath) + if let message = failure.message { #expect(error.message == message) } + #expect(error.message.first?.isUppercase == true) + #expect(error.message.hasSuffix(".")) + #expect(!error.message.isEmpty) + #expect(!error.description.contains("Stored-Spelling")) + #expect(!error.description.contains("/private/host")) + let encoded = try JSONEncoder().encode(error) + let json = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + #expect(json["code"] as? String == failure.code.rawValue) + #expect(json["path"] as? String == expectedPath?.string) + #expect(try JSONDecoder().decode(WorkspaceToolError.self, from: encoded) == error) + } + } + + @Test + func `real paths aliases and argument errors remain private`() async throws { + let root = try TestSupport.temporaryDirectory("ToolErrorPrivacy") + defer { TestSupport.remove(root) } + let local = try LocalFileSystem(root: root) + try await local.writeFile(path: "/Stored-Spelling", data: Data([0xff]), append: false) + try await local.createSymlink(path: "/requested", target: "Stored-Spelling") + try await local.createSymlink(path: "/outside", target: root.deletingLastPathComponent().appendingPathComponent("host-secret").path) + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: local)) + for (json, code, path) in [ + (#"{"path":"/requested"}"#, WorkspaceToolError.Code.illegalByteSequence, WorkspacePath("/requested")), + (#"{"path":"/stored-spelling"}"#, .notFound, "/stored-spelling"), + (#"{"path":"/outside"}"#, .invalidPath, "/outside"), + (#"{"path":"/missing"}"#, .notFound, "/missing"), + (#"{"path":"/Stored-Spelling/child"}"#, .notADirectory, "/Stored-Spelling/child"), + (#"{"path":"/"}"#, .isADirectory, "/"), + ] { + do { + _ = try await adapter.call(name: "workspace_read", arguments: Data(json.utf8)) + Issue.record("expected tool error") + } catch let error as WorkspaceToolError { + #expect(error.code == code) + #expect(error.path == path) + #expect(!error.message.contains("Stored-Spelling")) + #expect(!error.description.contains(root.path)) + #expect(!error.description.contains("host-secret")) + } + } + for (name, arguments, code) in [ + ("missing", "{}", WorkspaceToolError.Code.unknownTool), + ("workspace_read", #"{"path":3}"#, .invalidArguments), + ("workspace_search", #"{"pattern":"[","regular_expression":true}"#, .invalidArguments), + ("workspace_search", #"{"pattern":""}"#, .invalidArguments), + ("workspace_read", #"{"path":"/bad\u0000path"}"#, .invalidPath), + ("workspace_apply", #"{"edits":[{"kind":"write_text","path":"/missing-content"}]}"#, .invalidArguments), + ] { + do { + _ = try await adapter.call(name: name, arguments: Data(arguments.utf8)) + Issue.record("expected tool error") + } catch let error as WorkspaceToolError { + #expect(error.code == code) + #expect(error.path == nil) + } + } + } + + @Test(arguments: ["copy", "move"], ["memory", "local", "overlay"]) + func `subtree transfers explain invalid destinations`(kind: String, backend: String) async throws { + let root = try TestSupport.temporaryDirectory("ToolSubtreeError") + defer { TestSupport.remove(root) } + let fs: any FileSystem = switch backend { + case "local": try LocalFileSystem(root: root) + case "overlay": OverlayFileSystem(over: InMemoryFileSystem()) + default: InMemoryFileSystem() + } + try await fs.createFile(path: "/d/file", data: Data([1])) + try await fs.createSymlink(path: "/alias", target: "d") + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: fs)) + for destination in ["/d/new/x", "/alias/new/x"] { + let arguments = ["edits": [["kind": kind, "source": "/d", "path": destination]]] + await #expect(throws: WorkspaceToolError(code: .invalidPath, + message: "The destination is inside the source.")) { + _ = try await adapter.call(name: "workspace_apply", arguments: JSONEncoder().encode(arguments)) + } + } + } + + @Test + func `unknown tool messages include a bounded requested name`() async throws { + let adapter = WorkspaceToolAdapter(workspace: Workspace()) + for (name, displayed) in [("workspace_missing", "workspace_missing"), + (String(repeating: "x", count: 1000), String(repeating: "x", count: 128) + "…")] { + await #expect(throws: WorkspaceToolError(code: .unknownTool, + message: "Unknown workspace tool: \(displayed).")) { + _ = try await adapter.call(name: name, arguments: Data("{}".utf8)) + } + } + } + + @Test + func `read shrink reports a retryable conflict at the requested spelling`() async throws { + let memory = InMemoryFileSystem() + try await memory.writeFile(path: "/Stored-Spelling", data: Data("before".utf8), append: false) + try await memory.createSymlink(path: "/requested", target: "Stored-Spelling") + // Authorization runs after size metadata is obtained and before the range read. + let authorizer = PermissionAuthorizer { request in + if request.operation == .readFile { + try? await memory.writeFile(path: "/Stored-Spelling", data: Data(), append: false) + } + return .allow + } + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: AuthorizedFileSystem(base: memory, authorizer: authorizer))) + await #expect(throws: WorkspaceToolError(code: .conflict, path: "/requested", + message: "The file changed during the read; retry this page.")) { + _ = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/requested"}"#.utf8)) + } + } + + @Test + func `missing checkpoints do not blame the requested file`() async throws { + let workspace = Workspace() + try await workspace.writeText("/file.txt", "present") + let adapter = WorkspaceToolAdapter(workspace: workspace) + let arguments = Data("{\"path\":\"/file.txt\",\"checkpoint_id\":\"\(UUID())\"}".utf8) + await #expect(throws: WorkspaceToolError(code: .checkpointNotFound, + message: "The requested checkpoint or revision data was not found.")) { + _ = try await adapter.call(name: "workspace_read", arguments: arguments) + } + } + + @Test(arguments: [ + ("workspace_apply", #"{"edits":[{"path":"/file"}]}"#, "Missing required value at edits[0].kind."), + ("workspace_read", #"{"path":17}"#, "Expected a string at path."), + ("workspace_read", #"{"path":null}"#, "Expected a non-null value at path."), + ("workspace_read", #"{"path":"/file","offset":true}"#, "Expected an integer at offset."), + ("workspace_read", #"{"path":"/file","checkpoint_id":"secret-invalid-id"}"#, "The checkpoint ID must be a UUID."), + ("workspace_apply", #"{"edits":[{"path":"/file","kind":"secret-invalid-kind"}]}"#, "Unsupported value at edits[0].kind; use one of write_text, append_text, create_directory, remove, copy, move."), + ]) + func `argument diagnostics identify fields without values`(tool: String, arguments: String, message: String) async throws { + let adapter = WorkspaceToolAdapter(workspace: Workspace()) + await #expect(throws: WorkspaceToolError.invalidArguments(message)) { + _ = try await adapter.call(name: tool, arguments: Data(arguments.utf8)) + } + } + + @Test(arguments: ["workspace_glob", "workspace_search"], [false, true]) + func `glob and search root denials report only the requested root`(tool: String, explicitRoot: Bool) async throws { + let authorizer = PermissionAuthorizer { request in + .deny(message: "Private policy details for \(request.path?.string ?? "no path"): /Stored-Spelling") + } + let fs = AuthorizedFileSystem(base: InMemoryFileSystem(), authorizer: authorizer) + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: fs)) + let rootKey = tool == "workspace_glob" ? "current_directory" : "root" + let pattern = tool == "workspace_glob" ? "**/*" : "needle" + var arguments = ["pattern": pattern] + if explicitRoot { arguments[rootKey] = "/requested" } + do { + _ = try await adapter.call(name: tool, arguments: JSONEncoder().encode(arguments)) + Issue.record("expected policy denial") + } catch let error as WorkspaceToolError { + #expect(error.code == .permissionDenied) + #expect(error.path == (explicitRoot ? "/requested" : "/")) + #expect(error.message == "Workspace access is denied for this operation.") + #expect(!error.description.contains("Stored-Spelling")) + #expect(!error.description.contains("Private policy")) + } + } + + @Test + func `batch failures omit an ambiguous path and cancellation stays cancellation`() async throws { + let adapter = WorkspaceToolAdapter(workspace: Workspace()) + do { + _ = try await adapter.call(name: "workspace_apply", arguments: Data(#"{"edits":[{"kind":"write_text","path":"/new/file","content":"ok"},{"kind":"copy","source":"/missing","path":"/copy"}]}"#.utf8)) + Issue.record("expected missing source") + } catch let error as WorkspaceToolError { + #expect(error.code == .notFound) + #expect(error.path == nil) + } + #expect(!(await adapter.workspace.fileSystem.exists(path: "/new"))) + let fs = HookableFileSystem() + try await fs.writeFile(path: "/file", data: Data([1]), append: false) + await fs.onRead { _ in throw CancellationError() } + let cancellable = WorkspaceToolAdapter(workspace: Workspace(fileSystem: fs)) + await #expect(throws: CancellationError.self) { + _ = try await cancellable.call(name: "workspace_apply", arguments: Data(#"{"edits":[{"kind":"write_text","path":"/file","content":"new"}]}"#.utf8)) + } + } +} From ba7ed8fe955f2737bb33f7a7b93ecdd6668a3634 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 23:52:17 -0700 Subject: [PATCH 31/34] Never block a worker on a store lock, and hide in-flight write temporaries Store locks waited in a blocking flock on Dispatch's global queue. On Linux, where Swift's cooperative pool and the global workers are shared, enough contending stores in one process could occupy every worker while the current holder, an async task, waited for a thread to resume and release the lock. Locks are now taken with LOCK_NB, retrying after asynchronous 10 ms delays; uncontended acquisition is immediate, and lock order, cancellation and durability barriers are unchanged. A 256-store contention test hangs on the old code and finishes in about two seconds. LocalFileSystem's atomic writes stage a process-private temporary beside the destination. Change streams already ignored those names, but directory listings, walks and recursive copies did not. A shallow refresh racing a write could report the temporary as a user change, a listing could fail with ENOENT when it was renamed, and a recursive copy could duplicate it into the destination as a stray file. Listings, walks and copies now skip this process's in-flight temporaries; removal still sees them. --- Sources/Workspace/CheckpointStore.swift | 28 +++---- Sources/Workspace/FS/LocalFileSystem.swift | 7 +- .../CheckpointLockProgressTests.swift | 77 +++++++++++++++++++ .../ScopedObservationTests.swift | 33 +++++--- 4 files changed, 116 insertions(+), 29 deletions(-) create mode 100644 Tests/WorkspaceTests/CheckpointLockProgressTests.swift diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index a468439..9cdf75f 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -2689,27 +2689,19 @@ actor FileCheckpointStore: CheckpointStore { guard error == EWOULDBLOCK else { throw LockFileError(message: "could not acquire lock at \(path)") } - try await Self.waitForOSLock(fd, operation: operation, path: path) - break + // Dispatch's global workers are also finite (and shared with Swift on + // Linux). Blocking flock there can starve the async holder just as + // blocking this actor would. Suspend without occupying any worker. + // Deliberately ignore task cancellation: callers may already have + // changed the filesystem and must finish persisting that mutation. + await withCheckedContinuation { continuation in + DispatchQueue.global().asyncAfter(deadline: .now() + .milliseconds(10)) { + continuation.resume() + } + } } #endif return try await body() } -#if canImport(Darwin) || canImport(Glibc) - // flock may wait for another store or process. Never park a cooperative executor thread. - private nonisolated static func waitForOSLock(_ fd: Int32, operation: Int32, path: String) async throws { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - DispatchQueue.global().async { - while flock(fd, operation) != 0 { - if errno != EINTR { - continuation.resume(throwing: LockFileError(message: "could not acquire lock at \(path)")) - return - } - } - continuation.resume() - } - } - } -#endif } diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index e922a26..f4ffa7d 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -647,6 +647,8 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem records = try names(in: directory, consumingOffset: !consumedOffset).map { ($0, nil) } } return try (records ?? []).compactMap { name, suppliedInfo in + // In-flight atomic writes are this process's own; they surface only as their destination. + if LocalAtomicTemporaryName.matches(name) { return nil } let child = try Entry(parent: directory, storedName: name, path: path.appending(name)) let info: LocalStat do { info = try suppliedInfo ?? self.metadata(child) } @@ -764,7 +766,7 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem /// See ``FileSystem/listDirectory(path:)``. public func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { let directory = try openEntry(resolve(path, followFinal: true), flags: O_RDONLY | O_DIRECTORY) - return try names(in: directory).map { name in + return try names(in: directory).filter { !LocalAtomicTemporaryName.matches($0) }.map { name in let child = path.appending(name) return DirectoryEntry( name: name, info: fileInfo(try metadata(Entry(parent: directory, storedName: name, path: child)), path: child) @@ -1016,7 +1018,8 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem try check(mkdirat(destination.parent.value, destination.name, 0o755)) try validateCreated(destination, directory: true) let output = try openEntry(destination, flags: O_RDONLY | O_DIRECTORY) - for name in try names(in: input) { + // Another write's in-flight temporary is not part of the source tree. + for name in try names(in: input) where !LocalAtomicTemporaryName.matches(name) { try copyEntry( Entry(parent: input, storedName: name, path: source.path.appending(name)), to: Entry(parent: output, name: name, path: destination.path.appending(name)), recursive: true) diff --git a/Tests/WorkspaceTests/CheckpointLockProgressTests.swift b/Tests/WorkspaceTests/CheckpointLockProgressTests.swift new file mode 100644 index 0000000..bb25dda --- /dev/null +++ b/Tests/WorkspaceTests/CheckpointLockProgressTests.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Checkpoint lock progress") +struct CheckpointLockProgressTests { + // Only Linux reproduces the original deadlock: there Swift's pool and Dispatch's + // global workers are shared, while on macOS this passes even with blocking flock. + @Test + func `contending stores allow an async lock holder to resume`() async throws { + await #expect(processExitsWith: .success) { + // A watchdog on its own thread still runs if Dispatch and Swift workers stall. + Thread.detachNewThread { + Thread.sleep(forTimeInterval: 30) + exit(124) + } + let root = try TestSupport.temporaryDirectory("LockProgress") + defer { TestSupport.remove(root) } + let lock = root.appendingPathComponent("test.lock") + let entered = TestSignal() + let contenders = LockContenders() + let holder = FileCheckpointStore(rootDirectory: root) + let holding = Task { + try await holder.withExclusiveLock(at: lock) { + await entered.signal() + await contenders.waitUntilStarted() + // Give contended OS waits time to occupy workers before resuming. + try await Task.sleep(for: .milliseconds(200)) + } + } + await entered.wait() + try await withThrowingTaskGroup(of: Void.self) { group in + for index in 0..<256 { + group.addTask { + let store = FileCheckpointStore(rootDirectory: root) + // Persistence must still complete after an accepted edit + // cancels its task; a cancelled sleep must not busy-spin. + if index.isMultiple(of: 2) { withUnsafeCurrentTask { $0?.cancel() } } + await contenders.started() + try await store.withExclusiveLock(at: lock) { + await contenders.enter() + await Task.yield() + await contenders.leave() + } + } + } + try await group.waitForAll() + } + try await holding.value + #expect(await contenders.completed == 256) + } + } +} + +private actor LockContenders { + private var count = 0 + private var active = 0 + private var ready: CheckedContinuation? + private(set) var completed = 0 + + func started() { + count += 1 + if count == 256 { ready?.resume(); ready = nil } + } + func waitUntilStarted() async { + if count == 256 { return } + await withCheckedContinuation { ready = $0 } + } + func enter() { + active += 1 + #expect(active == 1) + } + func leave() { + active -= 1 + completed += 1 + } +} diff --git a/Tests/WorkspaceTests/ScopedObservationTests.swift b/Tests/WorkspaceTests/ScopedObservationTests.swift index 0b07b82..3fa04d2 100644 --- a/Tests/WorkspaceTests/ScopedObservationTests.swift +++ b/Tests/WorkspaceTests/ScopedObservationTests.swift @@ -57,6 +57,27 @@ struct ScopedObservationTests { } } + @Test func `in-flight atomic temporaries are hidden from listings and walks`() async throws { + let root = try TestSupport.temporaryDirectory() + defer { TestSupport.remove(root) } + let temporary = LocalAtomicTemporaryName.make() + let lookalike = ".workspace-atomic-" + UUID().uuidString + "-" + UUID().uuidString + for name in [temporary, lookalike, "file"] { + try Data("x".utf8).write(to: root.appendingPathComponent(name)) + } + let fs = try LocalFileSystem(root: root) + #expect(Set(try await fs.listDirectory(path: .root).map(\.name)) == [lookalike, "file"]) + var walked: [WorkspacePath] = [] + try await fs.walk(at: .root, skippingUnreadableDirectories: false) { info, _, _ in walked.append(info.path) } + #expect(Set(walked) == [.root, WorkspacePath(normalizing: "/" + lookalike), "/file"]) + // A recursive copy must not duplicate the temporary as a stray user file. + try FileManager.default.createDirectory(at: root.appendingPathComponent("src"), withIntermediateDirectories: false) + try Data("x".utf8).write(to: root.appendingPathComponent("src/" + temporary)) + try Data("y".utf8).write(to: root.appendingPathComponent("src/kept")) + try await fs.copy(from: "/src", to: "/dst", recursive: true) + #expect(try FileManager.default.contentsOfDirectory(atPath: root.appendingPathComponent("dst").path) == ["kept"]) + } + #if os(macOS) @Test func `FSEvents root item events are shallow only while the original directory remains`() throws { let container = try TestSupport.temporaryDirectory("RootItemIdentity").resolvingSymlinksInPath() @@ -128,7 +149,7 @@ struct ScopedObservationTests { let stats = await fs.base.statCount let reads = await fs.base.reads.count try await local.writeFile(path: "/x", data: Data("after".utf8), append: false) - guard case let .changes(change)? = await first(events, timeout: .seconds(3)) else { + guard case let .changes(change)? = await first(events, timeout: .seconds(30)) else { Issue.record("missing native external event"); return } #expect(change.touchedPaths == ["/x"]) @@ -142,13 +163,7 @@ struct ScopedObservationTests { } #endif - private func first(_ stream: AsyncStream, timeout: Duration = .seconds(5)) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { var iterator = stream.makeAsyncIterator(); return await iterator.next() } - group.addTask { try? await Task.sleep(for: timeout); return nil } - let event = await group.next() ?? nil - group.cancelAll() - return event - } + private func first(_ stream: AsyncStream, timeout: Duration = .seconds(30)) async -> WorkspaceEvent? { + await TestSupport.nextElement(in: stream, timeout: timeout) } } From 92b3fe0b09dd4fdfc7c6bd417ce19d23ba38fdd9 Mon Sep 17 00:00:00 2001 From: Zac White Date: Tue, 20 Oct 2026 23:52:17 -0700 Subject: [PATCH 32/34] Make event and retry tests robust to a saturated scheduler On a 4-CPU runner with the full suite in parallel, a 10 ms sleep was measured at 3.9 s, so 2-5 s deadlines for expected events expired before their waiters ran. Positive waits now allow 30 s and return as soon as the condition holds, and the timeout helper keeps an element that arrives alongside the timeout. Short "nothing else arrives" windows now start after a delivery marker, which makes them stricter. Retry tests wait for the retry state machine to finish, Limited tests wait for both byte and entry counts, and a 2 ms permission-cache race is gone. The bulk watch-retirement check compares process CPU time against same-run setup cost instead of a wall-clock bound, and still requires the removal to be reported. A quadratic implementation fails it. --- Tests/WorkspaceTests/ChangeScalingTests.swift | 21 +++-- .../WorkspaceTests/ChangeSoundnessTests.swift | 20 ++--- Tests/WorkspaceTests/EditAtomicityTests.swift | 16 +--- Tests/WorkspaceTests/EditPathTests.swift | 5 ++ Tests/WorkspaceTests/FileSystemAPITests.swift | 13 ++- Tests/WorkspaceTests/LimitedCacheTests.swift | 38 +++++--- .../LinuxFileChangeStreamTests.swift | 89 ++++++++++++++----- Tests/WorkspaceTests/PersistenceTests.swift | 28 +++--- .../WorkspaceTests/ReviewRevisionTests.swift | 19 ++-- Tests/WorkspaceTests/TestSupport.swift | 25 ++++++ Tests/WorkspaceTests/WorkspaceAPITests.swift | 23 ++--- 11 files changed, 186 insertions(+), 111 deletions(-) diff --git a/Tests/WorkspaceTests/ChangeScalingTests.swift b/Tests/WorkspaceTests/ChangeScalingTests.swift index 2427f74..cbdc003 100644 --- a/Tests/WorkspaceTests/ChangeScalingTests.swift +++ b/Tests/WorkspaceTests/ChangeScalingTests.swift @@ -173,8 +173,17 @@ struct ChangeScalingTests { #expect(await fs.base.reads.dropFirst(initialReads).allSatisfy { $0 == "/a" }) #expect(await fs.base.reads.count - initialReads <= 1) #expect(try await workspace.history().isEmpty) + // Finish any coalesced refresh before checking both work and silence. + // The later marker passes through the same signal consumer as all 20 hints. + try await fs.base.writeFile(path: "/marker", data: Data(), append: false) + fs.signal(.paths(["/marker": .entry])) + guard case let .changes(marker)? = await event(in: stream) else { + Issue.record("missing signal delivery fence"); return + } + #expect(marker.touchedPaths == ["/marker"]) + // The marker itself needs one additional stat. + #expect(await fs.base.statCount - initialStats <= 3) #expect(await event(in: stream, timeout: .milliseconds(150)) == nil) - #expect(await fs.base.statCount - initialStats <= 2) } @Test @@ -241,14 +250,8 @@ struct ChangeScalingTests { #expect(evicted.textDiffOmissions == [.init(path: "/b", reason: .observationCacheMiss)]) } - private func event(in stream: AsyncStream, timeout: Duration = .seconds(2)) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { var iterator = stream.makeAsyncIterator(); return await iterator.next() } - group.addTask { try? await Task.sleep(for: timeout); return nil } - let value = await group.next() ?? nil - group.cancelAll() - return value - } + private func event(in stream: AsyncStream, timeout: Duration = .seconds(30)) async -> WorkspaceEvent? { + await TestSupport.nextElement(in: stream, timeout: timeout) } } diff --git a/Tests/WorkspaceTests/ChangeSoundnessTests.swift b/Tests/WorkspaceTests/ChangeSoundnessTests.swift index 8922af6..aa7d6d1 100644 --- a/Tests/WorkspaceTests/ChangeSoundnessTests.swift +++ b/Tests/WorkspaceTests/ChangeSoundnessTests.swift @@ -224,10 +224,11 @@ struct ChangeSoundnessTests { } return nil } - group.addTask { try? await Task.sleep(for: .seconds(2)); return nil } + group.addTask { try? await Task.sleep(for: .seconds(30)); return nil } let value = await group.next() ?? nil group.cancelAll() - return value + let delivered = await group.next() ?? nil + return value ?? delivered } guard case let .paths(paths)? = signal else { Issue.record("expected scoped FSEvents notification, got \(String(describing: signal))") @@ -246,6 +247,7 @@ struct ChangeSoundnessTests { // No settling delay: the root's own creation can arrive with the file event. let stream = try await fs.changeSignals() try await fs.writeFile(path: "/x", data: Data("changed".utf8), append: false) + let (received, receipt) = AsyncStream.makeStream() let collector = Task { var combined: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] var rescanned = false @@ -254,11 +256,13 @@ struct ChangeSoundnessTests { case .rescan: rescanned = true case let .paths(paths): combined.merge(paths, uniquingKeysWith: max) } + if combined["/x"] != nil { receipt.yield(true) } } return (combined, rescanned) } defer { collector.cancel() } - // Unlike the first-event probe above, include delayed batches after /x. + // Start the short quiet window only after delivery, not after the write. + #expect(await TestSupport.nextElement(in: received) == true) try await Task.sleep(for: .seconds(2)) collector.cancel() let (paths, rescanned) = await collector.value @@ -269,14 +273,8 @@ struct ChangeSoundnessTests { } #endif - private func first(_ stream: AsyncStream, timeout: Duration = .seconds(5)) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { var it = stream.makeAsyncIterator(); return await it.next() } - group.addTask { try? await Task.sleep(for: timeout); return nil } - let v = await group.next() ?? nil - group.cancelAll() - return v - } + private func first(_ stream: AsyncStream, timeout: Duration = .seconds(30)) async -> WorkspaceEvent? { + await TestSupport.nextElement(in: stream, timeout: timeout) } } diff --git a/Tests/WorkspaceTests/EditAtomicityTests.swift b/Tests/WorkspaceTests/EditAtomicityTests.swift index 4b5ade3..2b55072 100644 --- a/Tests/WorkspaceTests/EditAtomicityTests.swift +++ b/Tests/WorkspaceTests/EditAtomicityTests.swift @@ -344,19 +344,7 @@ enum EditOperation: CaseIterable, Sendable { } func nextEditEvent( - in stream: AsyncStream, timeout: Duration = .seconds(2) + in stream: AsyncStream, timeout: Duration = .seconds(30) ) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { - var iterator = stream.makeAsyncIterator() - return await iterator.next() - } - group.addTask { - try? await Task.sleep(for: timeout) - return nil - } - let result = await group.next() ?? nil - group.cancelAll() - return result - } + await TestSupport.nextElement(in: stream, timeout: timeout) } diff --git a/Tests/WorkspaceTests/EditPathTests.swift b/Tests/WorkspaceTests/EditPathTests.swift index bdd92d4..1fab1f1 100644 --- a/Tests/WorkspaceTests/EditPathTests.swift +++ b/Tests/WorkspaceTests/EditPathTests.swift @@ -200,6 +200,11 @@ struct EditPathTests { // Exercise the refresh directly as well as any native invalidation. This // verifies baseline folding without depending on OS event delivery timing. await workspace.refreshFilesystemEvents() + try await fs.writeFile(path: "/marker", data: Data(), append: false) + guard case let .changes(marker)? = await nextEditEvent(in: events) else { + Issue.record("missing native delivery fence"); return + } + #expect(marker.touchedPaths == ["/marker"]) #expect(await nextEditEvent(in: events, timeout: .milliseconds(100)) == nil) } } diff --git a/Tests/WorkspaceTests/FileSystemAPITests.swift b/Tests/WorkspaceTests/FileSystemAPITests.swift index c89c55c..a333a8b 100644 --- a/Tests/WorkspaceTests/FileSystemAPITests.swift +++ b/Tests/WorkspaceTests/FileSystemAPITests.swift @@ -60,14 +60,19 @@ struct FileSystemAPITests { @Test func `authorization supports temporary approvals rules and bounded audit`() async throws { - let authorizer = PermissionAuthorizer(auditCapacity: 2) { _ in .allowFor(.milliseconds(2)) } + let authorizer = PermissionAuthorizer(auditCapacity: 2) { _ in .allowFor(.seconds(60)) } let request = PermissionRequest(operation: .readFile, path: "/a") _ = await authorizer.authorize(request) _ = await authorizer.authorize(request) #expect(await authorizer.auditLog().map(\.source) == [.handler, .temporaryCache]) - try await Task.sleep(for: .milliseconds(5)) - _ = await authorizer.authorize(request) - #expect(await authorizer.auditLog().count == 2) + _ = await authorizer.authorize(.init(operation: .readFile, path: "/b")) + #expect(await authorizer.auditLog().map(\.source) == [.temporaryCache, .handler]) + + // Exercise expiry without requiring two actor calls to fit inside 2 ms. + let expired = PermissionAuthorizer { _ in .allowFor(.zero) } + _ = await expired.authorize(request) + _ = await expired.authorize(request) + #expect(await expired.auditLog().map(\.source) == [.handler, .handler]) let rules = RuleBasedPermissionAuthorizer( rules: [.init(operations: [.readFile], pathPrefix: "/src", effect: .allow)] diff --git a/Tests/WorkspaceTests/LimitedCacheTests.swift b/Tests/WorkspaceTests/LimitedCacheTests.swift index 268c2fc..19ee796 100644 --- a/Tests/WorkspaceTests/LimitedCacheTests.swift +++ b/Tests/WorkspaceTests/LimitedCacheTests.swift @@ -389,19 +389,19 @@ struct LimitedCacheTests { let fs = LimitedFileSystem(base: base, limits: .init(maxTotalBytes: 100, maxEntryCount: 16)) try await fs.writeFile(path: "/a", data: Data(count: 3), append: false) try await base.writeFile(path: "/dir/file", data: Data(count: 5), append: false) - try await waitForBytes(8, fs: fs) + try await waitForBytes(8, entries: 3, fs: fs) try await checkTotals(fs, base: base) try await base.createHardLink(path: "/alias", target: "/dir/file") try await base.writeFile(path: "/alias", data: Data(count: 2), append: true) - try await waitForBytes(17, fs: fs) + try await waitForBytes(17, entries: 4, fs: fs) try await checkTotals(fs, base: base) try await base.move(from: "/dir", to: "/moved") try await base.createSymlink(path: "/dangling", target: "/absent") try await base.remove(path: "/a", recursive: false) - try await waitForBytes(14, fs: fs) + try await waitForBytes(14, entries: 4, fs: fs) try await checkTotals(fs, base: base) try await base.remove(path: "/moved", recursive: true) - try await waitForBytes(7, fs: fs) + try await waitForBytes(7, entries: 2, fs: fs) try await checkTotals(fs, base: base) #if os(Linux) if geteuid() != 0 { @@ -412,29 +412,45 @@ struct LimitedCacheTests { await #expect(throws: (any Error).self) { try await fs.revalidate() } try await base.setPermissions(path: "/locked", permissions: .defaultDirectory) try await base.writeFile(path: "/locked/file", data: Data(count: 2), append: false) - try await waitForBytes(9, fs: fs) + try await waitForBytes(9, entries: 4, fs: fs) try await Task.sleep(for: .milliseconds(100)) - try await waitForBytes(9, fs: fs) + try await waitForBytes(9, entries: 4, fs: fs) // After the permission event has been consumed, a second child write must // have its own watch; a stale shallow invalidation cannot hide a gap. try await base.writeFile(path: "/locked/file", data: Data(count: 3), append: false) - try await waitForBytes(10, fs: fs) + try await waitForBytes(10, entries: 4, fs: fs) try await checkTotals(fs, base: base) } #endif } #endif - private func waitForBytes(_ bytes: UInt64, fs: LimitedFileSystem, limit: UInt64 = 100) async throws { - let deadline = ContinuousClock.now + .seconds(3) + private func waitForBytes(_ bytes: UInt64, entries: Int? = nil, fs: LimitedFileSystem, limit: UInt64 = 100) async throws { + let deadline = ContinuousClock.now + .seconds(30) while true { do { try await fs.createFile(path: "/byte-probe", data: Data(count: Int(limit + 1))) Issue.record("quota probe unexpectedly succeeded") return } catch let error as FileSystemLimitError { - if error == .totalBytes(attempted: bytes + limit + 1, limit: limit) { return } - if ContinuousClock.now >= deadline { Issue.record("unexpected usage: \(error), expected \(bytes)"); return } + if error == .totalBytes(attempted: bytes + limit + 1, limit: limit) { + guard let entries else { return } + // File removal and directory retirement can arrive separately. + // Correct bytes alone do not prove that all entries converged. + let deep = WorkspacePath(normalizing: (0..<17).map { "entry-probe-\($0)" }.joined(separator: "/")) + do { + try await fs.createDirectory(path: deep, recursive: true) + Issue.record("entry quota probe unexpectedly succeeded") + return + } catch let entryError as FileSystemLimitError { + if entryError == .entryCount(attempted: entries + 17, limit: 16) { return } + if ContinuousClock.now >= deadline { + Issue.record("unexpected entry usage: \(entryError), expected \(entries)"); return + } + } + } else if ContinuousClock.now >= deadline { + Issue.record("unexpected usage: \(error), expected \(bytes)"); return + } } await Task.yield() } diff --git a/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift b/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift index a6d8eae..ff7d461 100644 --- a/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift +++ b/Tests/WorkspaceTests/LinuxFileChangeStreamTests.swift @@ -76,6 +76,7 @@ struct LinuxFileChangeStreamTests { #expect(await signals.waitForPath("/child", after: beforeRemoval)) // Include any queued IN_IGNORED notification after watch retirement. try await Task.sleep(for: .milliseconds(150)) + #expect(try await signals.fence(on: fs)) #expect(await signals.rescans == 0) } @@ -98,6 +99,7 @@ struct LinuxFileChangeStreamTests { try await fs.writeFile(path: userPath, data: Data("user file".utf8), append: false) #expect(await signals.waitForPath(userPath, after: 0)) try await Task.sleep(for: .milliseconds(150)) + #expect(try await signals.fence(on: fs)) #expect(await signals.rescans == 0) #expect(await signals.scopes["/x"] == .entry) #expect(await signals.scopes["/d"] == .shallow) @@ -122,6 +124,7 @@ struct LinuxFileChangeStreamTests { if !moveOut { #expect(await signals.waitForPath("/e", after: 0)) } // IN_IGNORED from our rm_watch arrives in a later batch. try await Task.sleep(for: .milliseconds(400)) + #expect(try await signals.fence(on: fs)) #expect(await signals.rescans == 0) #expect(await signals.scopes["/d"] == .recursive) if !moveOut { @@ -150,25 +153,43 @@ struct LinuxFileChangeStreamTests { } @Test - func `removing thousands of watched directories promptly emits an invalidation`() async throws { - let root = try TestSupport.temporaryDirectory("BulkDirectoryRemoval") - defer { TestSupport.remove(root) } - let bulk = root.appendingPathComponent("bulk", isDirectory: true) - try FileManager.default.createDirectory(at: bulk, withIntermediateDirectories: true) - for index in 0..<6000 { - try FileManager.default.createDirectory( - atPath: bulk.path + "/d\(index)", withIntermediateDirectories: false - ) + func `bulk watch retirement has bounded cost relative to registration`() async throws { + await #expect(processExitsWith: .success) { + Thread.detachNewThread { + Thread.sleep(forTimeInterval: 60) + exit(124) + } + // A child isolates process CPU time from other tests; CPU time excludes + // cgroup throttling and time waiting for a worker to deliver the signal. + // Compare against linear setup on the same filesystem in the same run. + let root = try TestSupport.temporaryDirectory("BulkDirectoryRemoval") + defer { TestSupport.remove(root) } + let bulk = root.appendingPathComponent("bulk", isDirectory: true) + let start = processCPUTime() + try FileManager.default.createDirectory(at: bulk, withIntermediateDirectories: true) + for index in 0..<6000 { + try FileManager.default.createDirectory( + atPath: bulk.path + "/d\(index)", withIntermediateDirectories: false + ) + } + let fs = try LocalFileSystem(root: root) + let stream = try await fs.changeSignals() + let signals = LinuxScopedSignals() + let consumer = Task { for await signal in stream { await signals.record(signal) } } + defer { consumer.cancel() } + let registered = processCPUTime() + try FileManager.default.removeItem(at: bulk) + // A marker after deletion proves the native queue drained the retirement + // work, rather than merely delivering the first event during removal. + #expect(try await signals.fence(on: fs)) + // Overflow may collapse the removal into a rescan; either must be reported. + let rescans = await signals.rescans + let reportedBulk = await signals.lastSeen["/bulk"] != nil + #expect(rescans > 0 || reportedBulk) + let retirement = processCPUTime() - registered + let registration = registered - start + #expect(retirement < max(0.05, registration) * 4) } - let stream = try DirectoryChangeStream.observe(root, recursive: true) - let counter = LinuxInvalidationCounter() - let consumer = Task { for await _ in stream { await counter.increment() } } - defer { consumer.cancel() } - let clock = ContinuousClock() - let start = clock.now - try FileManager.default.removeItem(at: bulk) - #expect(await counter.waitForChange(after: 0)) - #expect(start.duration(to: clock.now) < .seconds(2)) } @Test(.enabled(if: geteuid() != 0, "requires unprivileged filesystem permissions")) @@ -275,13 +296,29 @@ private actor LinuxScopedSignals { var rescans = 0 var scopes: [WorkspacePath: FileSystemChangeSignal.Scope] = [:] var lastSeen: [WorkspacePath: Int] = [:] + func fence(on fs: LocalFileSystem) async throws -> Bool { + // Native records preceding this write must be drained before its signal. + let marker = WorkspacePath(normalizing: "/fence-\(UUID().uuidString)") + let deadline = ContinuousClock.now + .seconds(30) + repeat { + let previous = count + try await fs.writeFile(path: marker, data: Data(), append: false) + while count == previous, ContinuousClock.now < deadline { + try await Task.sleep(for: .milliseconds(10)) + } + if lastSeen[marker] != nil { return true } + // A deletion batch can legitimately collapse paths into .rescan. + // Reissue the marker after that batch until a scoped receipt arrives. + } while ContinuousClock.now < deadline + return false + } func waitForRescan(after previous: Int = 0) async -> Bool { - let deadline = ContinuousClock.now.advanced(by: .seconds(2)) + let deadline = ContinuousClock.now.advanced(by: .seconds(30)) while ContinuousClock.now < deadline { if rescans > previous { return true } try? await Task.sleep(for: .milliseconds(10)) } - return false + return rescans > previous } func record(_ signal: FileSystemChangeSignal) { count += 1 @@ -295,12 +332,12 @@ private actor LinuxScopedSignals { } } func waitForPath(_ path: WorkspacePath, after previous: Int) async -> Bool { - let deadline = ContinuousClock.now.advanced(by: .seconds(2)) + let deadline = ContinuousClock.now.advanced(by: .seconds(30)) while ContinuousClock.now < deadline { if (lastSeen[path] ?? 0) > previous { return true } try? await Task.sleep(for: .milliseconds(10)) } - return false + return (lastSeen[path] ?? 0) > previous } } @@ -310,7 +347,7 @@ private actor LinuxInvalidationCounter { func waitForChange(after previous: Int) async -> Bool { let clock = ContinuousClock() - let deadline = clock.now.advanced(by: .seconds(2)) + let deadline = clock.now.advanced(by: .seconds(30)) while clock.now < deadline { if value > previous { return true } try? await Task.sleep(for: .milliseconds(10)) @@ -327,4 +364,10 @@ private actor LinuxInvalidationCounter { return value } } + +private func processCPUTime() -> Double { + var time = timespec() + precondition(clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &time) == 0) + return Double(time.tv_sec) + Double(time.tv_nsec) / 1_000_000_000 +} #endif diff --git a/Tests/WorkspaceTests/PersistenceTests.swift b/Tests/WorkspaceTests/PersistenceTests.swift index 43cada6..d929e9f 100644 --- a/Tests/WorkspaceTests/PersistenceTests.swift +++ b/Tests/WorkspaceTests/PersistenceTests.swift @@ -24,7 +24,11 @@ struct PersistenceTests { @Test func `checkpoint cursors exclude mutations appended after capture starts`() async throws { await #expect(processExitsWith: .success) { - DispatchQueue.global().asyncAfter(deadline: .now() + 10) { exit(124) } + // Keep the deadlock bound independent of Dispatch worker availability. + Thread.detachNewThread { + Thread.sleep(forTimeInterval: 60) + exit(124) + } let root = try TestSupport.temporaryDirectory("CaptureCursor") defer { TestSupport.remove(root) } for persistence in [Workspace.Persistence.memory, .directory(root)] { @@ -392,7 +396,11 @@ struct PersistenceTests { await #expect(processExitsWith: .success) { // An unstructured timeout cannot recover a blocked actor. Bound the child process // instead, so the original synchronous-flock deadlock fails without hanging the suite. - DispatchQueue.global().asyncAfter(deadline: .now() + 10) { exit(124) } + // Keep the deadlock bound independent of Dispatch worker availability. + Thread.detachNewThread { + Thread.sleep(forTimeInterval: 60) + exit(124) + } let filesRoot = try TestSupport.temporaryDirectory("ConcurrentCaptureFiles") let historyRoot = try TestSupport.temporaryDirectory("ConcurrentCaptureHistory") defer { TestSupport.remove(filesRoot); TestSupport.remove(historyRoot) } @@ -435,21 +443,9 @@ struct PersistenceTests { private func nextPersistenceEvent( in stream: AsyncStream, - timeout: Duration = .seconds(2) + timeout: Duration = .seconds(30) ) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { - var iterator = stream.makeAsyncIterator() - return await iterator.next() - } - group.addTask { - try? await Task.sleep(for: timeout) - return nil - } - let result = await group.next() ?? nil - group.cancelAll() - return result - } + await TestSupport.nextElement(in: stream, timeout: timeout) } private actor ReadCountingFileSystem: FileSystem { diff --git a/Tests/WorkspaceTests/ReviewRevisionTests.swift b/Tests/WorkspaceTests/ReviewRevisionTests.swift index 256da13..0c1a9ad 100644 --- a/Tests/WorkspaceTests/ReviewRevisionTests.swift +++ b/Tests/WorkspaceTests/ReviewRevisionTests.swift @@ -54,10 +54,17 @@ struct ReviewRevisionTests { let stats = await fs.base.statCount await fs.base.failStat(NSError(domain: NSPOSIXErrorDomain, code: Int(EACCES))) fs.signal(.rescan) - let deadline = ContinuousClock.now.advanced(by: .seconds(5)) - while await fs.base.statCount - stats < 3, ContinuousClock.now < deadline { + let deadline = ContinuousClock.now.advanced(by: .seconds(30)) + while ContinuousClock.now < deadline { + let stopped = await ws.filesystemRefreshFailures == 3 + let idle = await ws.filesystemRefreshTask == nil + if stopped && idle { break } try await Task.sleep(for: .milliseconds(10)) } + // statCount increments before the refresh resumes on Workspace. Wait for + // the retry state machine to finish before checking that it stays stopped. + #expect(await ws.filesystemRefreshFailures == 3) + #expect(await ws.filesystemRefreshTask == nil) #expect(await fs.base.statCount - stats == 3) try await Task.sleep(for: .milliseconds(500)) #expect(await fs.base.statCount - stats == 3) @@ -136,12 +143,6 @@ struct ReviewRevisionTests { } private func first(_ stream: AsyncStream) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { var iterator = stream.makeAsyncIterator(); return await iterator.next() } - group.addTask { try? await Task.sleep(for: .seconds(2)); return nil } - let event = await group.next() ?? nil - group.cancelAll() - return event - } + await TestSupport.nextElement(in: stream, timeout: .seconds(30)) } } diff --git a/Tests/WorkspaceTests/TestSupport.swift b/Tests/WorkspaceTests/TestSupport.swift index 2b2af8b..c6a45f0 100644 --- a/Tests/WorkspaceTests/TestSupport.swift +++ b/Tests/WorkspaceTests/TestSupport.swift @@ -2,6 +2,31 @@ import Foundation @testable import Workspace enum TestSupport { + /// Start the reader task before its deadline, and keep a delivered + /// element even if a delayed reader finishes after the timeout task. + static func nextElement( + in stream: AsyncStream, timeout: Duration = .seconds(30) + ) async -> Element? { + let (ready, started) = AsyncStream.makeStream() + return await withTaskGroup(of: Element?.self) { group in + group.addTask { + var iterator = stream.makeAsyncIterator() + started.yield() + started.finish() + return await iterator.next() + } + group.addTask { + for await _ in ready { break } + try? await Task.sleep(for: timeout) + return nil + } + let first = await group.next() ?? nil + group.cancelAll() + let second = await group.next() ?? nil + return first ?? second + } + } + static func temporaryDirectory(_ prefix: String = "WorkspaceTests") throws -> URL { let url = FileManager.default.temporaryDirectory .appendingPathComponent("\(prefix)-\(UUID().uuidString)", isDirectory: true) diff --git a/Tests/WorkspaceTests/WorkspaceAPITests.swift b/Tests/WorkspaceTests/WorkspaceAPITests.swift index 10e5f70..e9276c6 100644 --- a/Tests/WorkspaceTests/WorkspaceAPITests.swift +++ b/Tests/WorkspaceTests/WorkspaceAPITests.swift @@ -43,6 +43,13 @@ struct WorkspaceAPITests { let expected = try await workspace.writeText("/note", "once") #expect(await nextEvent(in: stream) == .changes(expected)) + // Fence native delivery with a later external write. A quiet timer alone + // could pass while the native refresh was still waiting for a worker. + try await workspace.fileSystem.writeFile(path: "/marker", data: Data(), append: false) + guard case let .changes(marker)? = await nextEvent(in: stream) else { + Issue.record("missing native delivery fence"); return + } + #expect(marker.touchedPaths == ["/marker"]) #expect(await nextEvent(in: stream, timeout: .milliseconds(200)) == nil) } @@ -179,19 +186,7 @@ struct WorkspaceAPITests { private func nextEvent( in stream: AsyncStream, - timeout: Duration = .seconds(2) + timeout: Duration = .seconds(30) ) async -> WorkspaceEvent? { - await withTaskGroup(of: WorkspaceEvent?.self) { group in - group.addTask { - var iterator = stream.makeAsyncIterator() - return await iterator.next() - } - group.addTask { - try? await Task.sleep(for: timeout) - return nil - } - let result = await group.next() ?? nil - group.cancelAll() - return result - } + await TestSupport.nextElement(in: stream, timeout: timeout) } From d179a4e4aae56c8cbdf0fa35ceac609262e84f47 Mon Sep 17 00:00:00 2001 From: Zac White Date: Wed, 21 Oct 2026 11:31:55 -0700 Subject: [PATCH 33/34] Authorize tool reads and hard links on the entry the caller named workspace_read resolved a symlink and read the resolved path, so AuthorizedFileSystem never saw a readFile for the requested spelling: a read deny on /blocked -> /public.txt was bypassed while readText enforced it. Tool reads now read and stat through the requested spelling, so wrappers authorize it and its target, and they check the target's resolution, identity, size and mtime around each range read; any change is the existing retry-this-page conflict. Resolution goes through the path resolver, so backends without real-path resolution, with or without symlinks, can serve tool reads again. Hard links now link a final symlink itself rather than its target on Local and Overlay, matching InMemory and LimitedFileSystem's projection. Previously an overlay could hard-link through a symlink to a regular file and exceed a byte quota. Authorization resolves a hard-link source the same way, so a deny rule can't be bypassed through an alias to a symlink in a protected directory, and dangling symlinks can be linked through AuthorizedFileSystem. The contract is documented on createHardLink. On Darwin, a hard-linked symlink's stored spelling is verified against the directory listing, because getattrlist can report another link's name. Scoped observation refreshes no longer stat a descendant through an ancestor that has become a symlink or a non-directory. A delayed signal beneath a replaced directory used to insert the link target's metadata as a phantom entry, followed by a phantom deletion on rescan. Each unique ancestor is checked at most once per refresh. --- README.md | 2 + Sources/Workspace/FS/FileSystem.swift | 5 +- Sources/Workspace/FS/LocalFileSystem.swift | 2 +- Sources/Workspace/FS/LocalPathSpelling.swift | 10 + Sources/Workspace/FS/OverlayFileSystem.swift | 2 +- Sources/Workspace/FilesystemObservation.swift | 40 +++- Sources/Workspace/Support/Permissions.swift | 9 +- Sources/Workspace/WorkspaceRevision.swift | 45 +++- Sources/Workspace/WorkspaceToolAdapter.swift | 3 +- Sources/Workspace/WorkspaceToolError.swift | 4 + .../WorkspaceTests/ParentCreationTests.swift | 36 ++++ Tests/WorkspaceTests/ReviewFixTests.swift | 199 ++++++++++++++++++ .../ScopedObservationTests.swift | 41 ++++ 13 files changed, 384 insertions(+), 14 deletions(-) create mode 100644 Tests/WorkspaceTests/ReviewFixTests.swift diff --git a/README.md b/README.md index 317c74a..74746f4 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,8 @@ Without cached contents, observation relies on metadata: an identical rewrite ca | `AuthorizedFileSystem(base:authorizer:)` | Checks every operation against a policy. | | `LimitedFileSystem(base:limits:)` | Rejects mutations that would exceed byte, entry or per-write limits. | +Hard links follow intermediate symlinks in their source path. If the source's final entry is a symlink, the hard link shares that symlink itself, even when its target is missing. It does not link the target file. + ```swift let local = try LocalFileSystem(root: projectURL) let limited = LimitedFileSystem( diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index f1ed8a6..ed1e3c9 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -197,7 +197,10 @@ public protocol FileSystem: AnyObject, Sendable { func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws /// Creates a symbolic link at `path` pointing to `target`. func createSymlink(path: WorkspacePath, target: String) async throws - /// Creates a hard link at `path` pointing to `target`. + /// Creates a hard link at `path` pointing to the entry at `target`. + /// Intermediate symlinks in `target` are followed. A final symlink is linked + /// itself, without dereferencing it, like `linkat` without `AT_SYMLINK_FOLLOW`. + /// A dangling final symlink can also be linked. func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws /// Reads the target of the symbolic link at `path`. func readSymlink(path: WorkspacePath) async throws -> String diff --git a/Sources/Workspace/FS/LocalFileSystem.swift b/Sources/Workspace/FS/LocalFileSystem.swift index f4ffa7d..4d601b1 100644 --- a/Sources/Workspace/FS/LocalFileSystem.swift +++ b/Sources/Workspace/FS/LocalFileSystem.swift @@ -1073,7 +1073,7 @@ public final class LocalFileSystem: FileSystem, TraversingFileSystem, PreviewSem /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { let config = try requireConfiguration() - let source = try resolve(target, followFinal: true, configuration: config) + let source = try resolve(target, followFinal: false, configuration: config) let destination = try resolve(path, followFinal: false, createParents: true, creating: true, configuration: config) // No AT_SYMLINK_FOLLOW: a concurrent final-component swap must not dereference a link. try check(linkat(source.parent.value, source.name, destination.parent.value, destination.name, 0)) diff --git a/Sources/Workspace/FS/LocalPathSpelling.swift b/Sources/Workspace/FS/LocalPathSpelling.swift index fe8a8ff..c5f5d43 100644 --- a/Sources/Workspace/FS/LocalPathSpelling.swift +++ b/Sources/Workspace/FS/LocalPathSpelling.swift @@ -145,6 +145,16 @@ final class LocalPathSpelling: @unchecked Sendable { throw posix(EIO) } guard bytes[start..<(start + count - 1)].elementsEqual(name.utf8) else { + // Darwin can return another hard link's name for a symlink vnode. + // Verify the actual directory entry before rejecting that spelling. + var info = stat() + if fstatat(parent, name, &info, AT_SYMLINK_NOFOLLOW) == 0, + info.st_mode & mode_t(S_IFMT) == mode_t(S_IFLNK), info.st_nlink > 1 { + try Self.checkListing(name: name, exists: true, creating: creating, afterCreate: afterCreate) { + try Self.listNames(parent) + } + return + } throw Self.spellingMismatch(creating: creating, afterCreate: afterCreate) } } diff --git a/Sources/Workspace/FS/OverlayFileSystem.swift b/Sources/Workspace/FS/OverlayFileSystem.swift index 5024f92..f2a94b3 100644 --- a/Sources/Workspace/FS/OverlayFileSystem.swift +++ b/Sources/Workspace/FS/OverlayFileSystem.swift @@ -419,7 +419,7 @@ public actor OverlayFileSystem: FileSystem, PreviewSemanticsFileSystem { /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { let createsParents = await previewSemantics(at: path).createsWriteParents - let target = try await resolveMergedPath(target) + let target = try await resolveMergedPath(target, followFinal: false) guard let targetInfo = await mergedEntry(target) else { throw posixError(ENOENT) } diff --git a/Sources/Workspace/FilesystemObservation.swift b/Sources/Workspace/FilesystemObservation.swift index 100cbb0..93ea0ca 100644 --- a/Sources/Workspace/FilesystemObservation.swift +++ b/Sources/Workspace/FilesystemObservation.swift @@ -115,11 +115,41 @@ struct FilesystemObservation: Sendable { } } var recursiveRoots: [WorkspacePath] = [] + var verifiedDirectories: Set = [] + // Check top-down: even stat follows intermediate symlinks. Entries + // already refreshed in this batch take precedence over the old index. + func hasDirectoryAncestors(_ path: WorkspacePath) async throws -> Bool { + guard !path.isRoot else { return true } + var ancestors: [WorkspacePath] = [] + var ancestor = path.parent + while !ancestor.isRoot { + ancestors.append(ancestor) + ancestor = ancestor.parent + } + for ancestor in ancestors.reversed() { + if let refreshed = next.entries[ancestor] { + guard refreshed.info.kind == .directory else { return false } + } else { + guard before.entries[ancestor] == nil, + !recursiveRoots.contains(where: { Snapshot.contains($0, ancestor) }), + metadata.entries[ancestor]?.info.kind == .directory + else { return false } + if !verifiedDirectories.contains(ancestor) { + guard try await Snapshot.infoIfPresent(at: ancestor, on: fs)?.kind == .directory + else { return false } + verifiedDirectories.insert(ancestor) + } + } + } + return true + } func captureRecursive(_ root: WorkspacePath) async throws { for path in subtree(root) { before.entries[path] = metadata.entries[path] } - let captured = try await MetadataIndex.capture(from: fs, at: root) - next.entries.merge(captured.entries) { _, new in new } - checkRacyContents.formUnion(captured.entries.keys) + if try await hasDirectoryAncestors(root) { + let captured = try await MetadataIndex.capture(from: fs, at: root) + next.entries.merge(captured.entries) { _, new in new } + checkRacyContents.formUnion(captured.entries.keys) + } recursiveRoots.append(root) } func captureEntry(_ info: FileInfo) async throws { @@ -133,6 +163,10 @@ struct FilesystemObservation: Sendable { if recursiveRoots.contains(where: { Snapshot.contains($0, path) }) { continue } let scope = affected[path]! let old = metadata.entries[path] + guard try await hasDirectoryAncestors(path) else { + try await captureRecursive(path) + continue + } let info = try await Snapshot.infoIfPresent(at: path, on: fs) let replacedDirectory = old?.info.kind == .directory && (info?.kind != .directory || old?.info.identity != info?.identity) diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 8d2ab87..de2f85e 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -494,7 +494,14 @@ public final class AuthorizedFileSystem: FileSystem, PreviewSemanticsFileSystem, if let path = request.destinationPath { resolved.destinationPath = try await resolvedPath(path, followFinal: false) } - case .createSymlink, .createHardLink: + case .createHardLink: + if let path = request.path { + resolved.path = try await resolvedPath(path, followFinal: false) + } + if let path = request.destinationPath { + resolved.destinationPath = try await resolvedPath(path, followFinal: false) + } + case .createSymlink: if let path = request.path { resolved.path = try await resolvedPath(path, followFinal: false) } diff --git a/Sources/Workspace/WorkspaceRevision.swift b/Sources/Workspace/WorkspaceRevision.swift index 4021c4d..0ccb44a 100644 --- a/Sources/Workspace/WorkspaceRevision.swift +++ b/Sources/Workspace/WorkspaceRevision.swift @@ -124,18 +124,53 @@ extension Workspace { var readRange: @Sendable (UInt64, Int?) async throws -> Data } - /// Resolve once for metadata and all reads needed to decode one UTF-8 page. + /// Keep one file identity across all reads needed to decode a UTF-8 page. func resolvedReadableFile(at path: WorkspacePath, revision: Revision) async throws -> ResolvedReadableFile { switch revision { case .current: - let resolvedPath = try await filesystem.resolveRealPath(path: path) - let metadata = try await filesystem.stat(path: resolvedPath) + let filesystem = filesystem + let features = await filesystem.capabilities() + let resolver = FileSystemPathResolver(base: filesystem) + @Sendable func currentMetadata() async throws -> (WorkspacePath, FileInfo) { + // The requested spelling's stat policy also governs the reported totalBytes. + let requested = try await filesystem.stat(path: path) + let resolved = try await resolver.resolve(path, features: features, followFinal: true) + let info = resolved == path ? requested : try await filesystem.stat(path: resolved) + return (resolved, info) + } + let (resolvedPath, metadata) = try await currentMetadata() guard metadata.kind == .file else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EISDIR)) } - let filesystem = filesystem + let conflict = WorkspaceToolError.readConflict(path: path) + @Sendable func validate() async throws { + do { + let (currentPath, current) = try await currentMetadata() + guard currentPath == resolvedPath else { throw conflict } + guard current.kind == .file, current.identity == metadata.identity, + current.size == metadata.size, current.modificationDate == metadata.modificationDate + else { throw conflict } + } catch { + let failure = error as NSError + if failure.domain == NSPOSIXErrorDomain, + [Int(ENOENT), Int(ENOTDIR), Int(ELOOP)].contains(failure.code) { throw conflict } + throw error + } + } return ResolvedReadableFile(size: metadata.size, readRange: { offset, length in - try await filesystem.readFile(path: resolvedPath, offset: offset, length: length) + // Check both sides to detect changes between ranges and during this read. + try await validate() + let data: Data + do { + // Preserve the caller's spelling so wrappers authorize both it and + // its resolved target, just as an ordinary Workspace read does. + data = try await filesystem.readFile(path: path, offset: offset, length: length) + } catch { + try await validate() + throw error + } + try await validate() + return data }) case let .checkpoint(id): let access = try await checkpointAccess(id) diff --git a/Sources/Workspace/WorkspaceToolAdapter.swift b/Sources/Workspace/WorkspaceToolAdapter.swift index 7a84039..007de29 100644 --- a/Sources/Workspace/WorkspaceToolAdapter.swift +++ b/Sources/Workspace/WorkspaceToolAdapter.swift @@ -198,8 +198,7 @@ public actor WorkspaceToolAdapter { hasMore: offset < totalBytes && UInt64(data.count) < totalBytes - offset, offset: offset, leadingContext: leadingContext) guard end > 0 || offset >= totalBytes else { - throw WorkspaceToolError(code: .conflict, path: request.path, - message: "The file changed during the read; retry this page.") + throw WorkspaceToolError.readConflict(path: request.path) } // The read length was bounded by totalBytes - offset, so this cannot overflow. let nextOffset = offset + UInt64(end) diff --git a/Sources/Workspace/WorkspaceToolError.swift b/Sources/Workspace/WorkspaceToolError.swift index ef567f4..7bfe263 100644 --- a/Sources/Workspace/WorkspaceToolError.swift +++ b/Sources/Workspace/WorkspaceToolError.swift @@ -58,6 +58,10 @@ public struct WorkspaceToolError: Error, Sendable, Codable, Equatable, CustomStr Self(code: .invalidArguments, path: nil, message: message) } + static func readConflict(path: WorkspacePath) -> Self { + Self(code: .conflict, path: path, message: "The file changed during the read; retry this page.") + } + init(mapping error: any Error, requestedPaths: [WorkspacePath]) { // Never substitute a backend-reported spelling. Even a valid virtual path // may disclose a resolved target the caller did not request. diff --git a/Tests/WorkspaceTests/ParentCreationTests.swift b/Tests/WorkspaceTests/ParentCreationTests.swift index 8c5456a..dad78c6 100644 --- a/Tests/WorkspaceTests/ParentCreationTests.swift +++ b/Tests/WorkspaceTests/ParentCreationTests.swift @@ -56,6 +56,42 @@ struct ParentCreationTests { } } + @Test(arguments: ["file", "missing"], ["local", "memory", "overlay", "authorized", "mounted", "limited local"]) + func `hard links preserve the final symlink across backends`(target: String, backend: String) async throws { + let root = try TestSupport.temporaryDirectory("HardLinkSymlink") + defer { TestSupport.remove(root) } + let lower = InMemoryFileSystem() + let fs: any FileSystem + switch backend { + case "local": fs = try LocalFileSystem(root: root) + case "overlay": fs = OverlayFileSystem(over: lower) + case "authorized": fs = AuthorizedFileSystem(base: lower, authorizer: PermissionAuthorizer { _ in .allow }) + case "mounted": fs = MountedFileSystem(base: InMemoryFileSystem(), mounts: [.init(mountPoint: "/mount", fileSystem: lower)]) + case "limited local": fs = LimitedFileSystem(base: try LocalFileSystem(root: root), limits: .init(maxTotalBytes: 8)) + default: fs = lower + } + let prefix: WorkspacePath = backend == "mounted" ? "/mount" : .root + let setup: any FileSystem = backend == "overlay" ? lower : fs + try await setup.createDirectory(path: prefix.appending("source"), recursive: true) + try await setup.writeFile(path: prefix.appending("source/file"), data: Data("contents".utf8), append: false) + try await setup.createSymlink(path: prefix.appending("source/link"), target: target) + try await setup.createSymlink(path: prefix.appending("alias"), target: "source") + let lowerBefore = try await Snapshot.capture(from: lower) + let hard = prefix.appending("new/deep/hard") + try await fs.createHardLink(path: hard, target: prefix.appending("alias/link")) + let info = try await fs.stat(path: hard) + #expect(info.kind == .symlink) + await #expect(throws: (any Error).self) { try await fs.stat(path: prefix.appending("new/deep/HARD")) } + #expect(try await fs.readSymlink(path: hard) == target) + #expect(info.identity == (try await fs.stat(path: prefix.appending("source/link"))).identity) + #expect(try await fs.stat(path: prefix.appending("new/deep")).kind == .directory) + if backend == "overlay" { #expect(try await Snapshot.capture(from: lower).entry == lowerBefore.entry) } + if let limited = fs as? LimitedFileSystem { + try await limited.revalidate() + await #expect(throws: FileSystemLimitError.self) { try await limited.createFile(path: "/extra", data: Data([1])) } + } + } + @Test(arguments: cases) func `creation matches disk including parents permissions and errors`(operation: Operation, route: String) async throws { let root = try TestSupport.temporaryDirectory("ParentCreation") diff --git a/Tests/WorkspaceTests/ReviewFixTests.swift b/Tests/WorkspaceTests/ReviewFixTests.swift new file mode 100644 index 0000000..130bdbb --- /dev/null +++ b/Tests/WorkspaceTests/ReviewFixTests.swift @@ -0,0 +1,199 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("PR review regressions") +struct ReviewFixTests { + @Test(arguments: [WorkspacePath("/blocked"), WorkspacePath("/public.txt")], [false, true]) + func toolReadAuthorizesBothSpellings(denied: WorkspacePath, limited: Bool) async throws { + let base = InMemoryFileSystem() + try await base.writeFile(path: "/public.txt", data: Data("public".utf8), append: false) + try await base.createSymlink(path: "/blocked", target: "public.txt") + let authorized = AuthorizedFileSystem(base: base, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.readFile], pathPrefix: denied, effect: .deny(message: "blocked")), + .init(operations: [.readFile, .stat, .resolveRealPath], pathPrefix: "/", effect: .allow), + ])) + let fs: any FileSystem = limited ? LimitedFileSystem(base: authorized, limits: .init(maxTotalBytes: 100)) : authorized + let workspace = Workspace(fileSystem: fs) + await #expect(throws: WorkspaceError.self) { try await workspace.readText("/blocked") } + await #expect(throws: WorkspaceError.self) { try await workspace.readData(from: "/blocked", offset: 0, length: 3) } + let adapter = WorkspaceToolAdapter(workspace: workspace) + do { + _ = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/blocked"}"#.utf8)) + Issue.record("tool read bypassed read authorization") + } catch let error as WorkspaceToolError { + #expect(error.code == .permissionDenied) + #expect(error.path == "/blocked") + } + } + + @Test func hardLinkAuthorizationDeniesResolvedSymlinkEntry() async throws { + let base = InMemoryFileSystem() + try await base.createDirectory(path: "/protected", recursive: true) + try await base.writeFile(path: "/public/file", data: Data("public".utf8), append: false) + try await base.createSymlink(path: "/protected/link", target: "../public/file") + try await base.createSymlink(path: "/alias", target: "protected") + let fs = AuthorizedFileSystem(base: base, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.createHardLink], pathPrefix: "/protected", effect: .deny(message: "protected")), + .init(operations: [.createHardLink], pathPrefix: "/", effect: .allow), + ])) + do { + try await fs.createHardLink(path: "/hard", target: "/alias/link") + Issue.record("hard link bypassed the protected symlink entry") + } catch let WorkspaceError.accessDenied(operation, _) { + #expect(operation == PermissionOperation.createHardLink.rawValue) + } + #expect(!(await base.exists(path: "/hard"))) + } + + @Test func toolReadAuthorizesRequestedMetadata() async throws { + let base = InMemoryFileSystem() + try await base.writeFile(path: "/public.txt", data: Data("public".utf8), append: false) + try await base.createSymlink(path: "/blocked", target: "public.txt") + let fs = AuthorizedFileSystem(base: base, authorizer: RuleBasedPermissionAuthorizer(rules: [ + .init(operations: [.stat], pathPrefix: "/blocked", effect: .deny(message: "metadata denied")), + .init(operations: [.readFile, .stat, .resolveRealPath], pathPrefix: "/", effect: .allow), + ])) + let workspace = Workspace(fileSystem: fs) + #expect(try await workspace.readText("/blocked") == "public") + do { + _ = try await WorkspaceToolAdapter(workspace: workspace).call( + name: "workspace_read", arguments: Data(#"{"path":"/blocked"}"#.utf8)) + Issue.record("tool read disclosed metadata despite requested-path stat denial") + } catch let error as WorkspaceToolError { + #expect(error.code == .permissionDenied) + #expect(error.path == "/blocked") + } + } + + @Test(arguments: [false, true]) + func toolReadWithoutRealPathResolution(symlinks: Bool) async throws { + let base = InMemoryFileSystem() + try await base.writeFile(path: "/dir/file", data: Data("ordinary read".utf8), append: false) + try await base.createSymlink(path: "/dir/link", target: "file") + try await base.createSymlink(path: "/alias", target: "dir") + let fs: any FileSystem = symlinks ? SymlinkOnlyFileSystem(base: base) : RequiredOperationsFileSystem(base: base) + let path: WorkspacePath = symlinks ? "/alias/link" : "/dir/file" + let workspace = Workspace(fileSystem: fs) + #expect(try await workspace.readText(path) == "ordinary read") + let result = try await WorkspaceToolAdapter(workspace: workspace).call( + name: "workspace_read", arguments: Data("{\"path\":\"\(path)\",\"offset\":2,\"max_bytes\":4}".utf8)) + let page = try JSONDecoder().decode(ReadPage.self, from: result) + #expect(page.content == "dina") + #expect(page.totalBytes == 13) + #expect(page.nextOffset == 6) + } + + @Test(arguments: ["retarget", "replace", "resize", "rewrite"]) + func toolReadRejectsChangesDuringRead(mutation: String) async throws { + let memory = InMemoryFileSystem() + // Mutate after the adapter has measured the file, before the read completes. + try await memory.writeFile(path: "/first", data: Data("éabcd".utf8), append: false) + try await memory.writeFile(path: "/second", data: Data("øwxyz".utf8), append: false) + try await memory.createSymlink(path: "/link", target: "first") + let hooked = HookableFileSystem(base: memory) + await hooked.onRead { _ in + switch mutation { + case "retarget": + try await memory.remove(path: "/link", recursive: false) + try await memory.createSymlink(path: "/link", target: "second") + case "replace": + try await memory.remove(path: "/first", recursive: false) + try await memory.writeFile(path: "/first", data: Data("øwxyz".utf8), append: false) + case "resize": try await memory.writeFile(path: "/first", data: Data("longer".utf8), append: true) + default: try await memory.writeFile(path: "/first", data: Data("øwxyz".utf8), append: false) + } + } + let fs = ResolvingReadFileSystem(base: hooked, resolver: memory) + let adapter = WorkspaceToolAdapter(workspace: Workspace(fileSystem: fs)) + do { + _ = try await adapter.call(name: "workspace_read", arguments: Data(#"{"path":"/link","offset":0,"max_bytes":4}"#.utf8)) + Issue.record("changed file returned a successful page") + } catch let error as WorkspaceToolError { + #expect(error.code == .conflict) + #expect(error.path == "/link") + #expect(error.message == "The file changed during the read; retry this page.") + } + } + + @Test(arguments: [false, true]) + func resolvedReadRejectsSymlinkRetargetBetweenRanges(symlinksOnly: Bool) async throws { + let fs = InMemoryFileSystem() + try await fs.writeFile(path: "/first", data: Data("abcdef".utf8), append: false) + try await fs.writeFile(path: "/second", data: Data("uvwxyz".utf8), append: false) + try await fs.createSymlink(path: "/link", target: "first") + let reader: any FileSystem = symlinksOnly ? SymlinkOnlyFileSystem(base: fs) : fs + let file = try await Workspace(fileSystem: reader).resolvedReadableFile(at: "/link", revision: .current) + #expect(try await file.readRange(0, 3) == Data("abc".utf8)) + try await fs.remove(path: "/link", recursive: false) + try await fs.createSymlink(path: "/link", target: "second") + do { + _ = try await file.readRange(3, 3) + Issue.record("retarget between ranges was not reported") + } catch let error as WorkspaceToolError { + #expect(error.code == .conflict) + #expect(error.message == "The file changed during the read; retry this page.") + } + } + + @Test(arguments: [false, true]) + func overlayHardLinkToSymlinkKeepsQuotaExact(upperSource: Bool) async throws { + let lower = InMemoryFileSystem() + let overlay = OverlayFileSystem(over: lower) + let source: any FileSystem = upperSource ? overlay : lower + try await source.writeFile(path: "/file", data: Data(repeating: 65, count: 10), append: false) + try await source.createSymlink(path: "/link", target: "file") + let limited = LimitedFileSystem(base: overlay, limits: .init(maxTotalBytes: 10)) + try await limited.createHardLink(path: "/hard", target: "/link") + #expect(try await overlay.stat(path: "/hard").kind == .symlink) + let entries = try await overlay.listDirectory(path: .root) + #expect(entries.filter { $0.info.kind == .file }.reduce(UInt64(0)) { $0 + $1.info.size } == 10) + await #expect(throws: FileSystemLimitError.self) { try await limited.createFile(path: "/extra", data: Data([1])) } + try await limited.revalidate() + await #expect(throws: FileSystemLimitError.self) { try await limited.createFile(path: "/extra", data: Data([1])) } + } + + private struct ReadPage: Decodable { + var content: String + var totalBytes: UInt64 + var nextOffset: UInt64 + } +} + +// Deliberately implement only required operations. Capabilities, resolution and +// range reads use the FileSystem defaults, as a minimal third-party backend does. +private protocol RequiredOperationsForwarding: FileSystem { var base: any FileSystem { get } } +private extension RequiredOperationsForwarding { + func stat(path: WorkspacePath) async throws -> FileInfo { try await base.stat(path: path) } + func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { try await base.listDirectory(path: path) } + func readFile(path: WorkspacePath) async throws -> Data { try await base.readFile(path: path) } + func exists(path: WorkspacePath) async -> Bool { await base.exists(path: path) } + func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + try await base.glob(pattern: pattern, currentDirectory: currentDirectory) + } + func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await base.writeFile(path: path, data: data, append: append) } + func createDirectory(path: WorkspacePath, recursive: Bool) async throws { try await base.createDirectory(path: path, recursive: recursive) } + func remove(path: WorkspacePath, recursive: Bool) async throws { try await base.remove(path: path, recursive: recursive) } + func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { try await base.move(from: sourcePath, to: destinationPath) } + func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { + try await base.copy(from: sourcePath, to: destinationPath, recursive: recursive) + } +} +private final class RequiredOperationsFileSystem: RequiredOperationsForwarding { + let base: any FileSystem + init(base: any FileSystem) { self.base = base } +} +private final class ResolvingReadFileSystem: RequiredOperationsForwarding { + let base: any FileSystem + let resolver: any FileSystem + init(base: any FileSystem, resolver: any FileSystem) { self.base = base; self.resolver = resolver } + func capabilities() async -> FileSystemFeatures { [.realPathResolution] } + func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { try await resolver.resolveRealPath(path: path) } +} + +private final class SymlinkOnlyFileSystem: RequiredOperationsForwarding { + let base: any FileSystem + init(base: any FileSystem) { self.base = base } + func capabilities() async -> FileSystemFeatures { [.symlinks] } + func readSymlink(path: WorkspacePath) async throws -> String { try await base.readSymlink(path: path) } +} diff --git a/Tests/WorkspaceTests/ScopedObservationTests.swift b/Tests/WorkspaceTests/ScopedObservationTests.swift index 3fa04d2..2a690de 100644 --- a/Tests/WorkspaceTests/ScopedObservationTests.swift +++ b/Tests/WorkspaceTests/ScopedObservationTests.swift @@ -7,6 +7,47 @@ import CoreServices @Suite("Scoped filesystem observation") struct ScopedObservationTests { + @Test func scopedRefreshStatsEachSharedAncestorOnce() async throws { + let fs = HookableFileSystem() + let paths = (0..<50).map { WorkspacePath(normalizing: "/a/b/c/d/file-\($0)") } + for path in paths { try await fs.writeFile(path: path, data: Data("old".utf8), append: false) } + var observation = try await FilesystemObservation.start(on: fs, budget: 0) + for path in paths { try await fs.writeFile(path: path, data: Data("changed".utf8), append: false) } + await fs.resetMetadataCounts() + let changes = try await observation.refresh(on: fs, + signal: .paths(Dictionary(uniqueKeysWithValues: paths.map { ($0, .entry) }))) + #expect(Set(changes.touchedPaths) == Set(paths)) + // Fifty leaves and four unique ancestors; the root needs no lookup. + #expect(await fs.statCount == 54) + #expect(await fs.listings.isEmpty) + #expect(try await observation.refresh(on: fs, signal: .rescan).changes.isEmpty) + } + + @Test(arguments: ["symlink", "file", "missing"], ["delayed", "same batch", "stale ancestor"]) + func `descendant invalidations never traverse replaced ancestors`(replacement: String, delivery: String) async throws { + let fs = InMemoryFileSystem() + try await fs.writeFile(path: "/dir/nested/file", data: Data("old".utf8), append: false) + try await fs.writeFile(path: "/other/nested/file", data: Data("other".utf8), append: false) + var observation = try await FilesystemObservation.start(on: fs, budget: 1024) + try await fs.remove(path: "/dir", recursive: true) + if replacement == "symlink" { try await fs.createSymlink(path: "/dir", target: "other") } + if replacement == "file" { try await fs.writeFile(path: "/dir", data: Data("leaf".utf8), append: false) } + let descendants: [WorkspacePath: FileSystemChangeSignal.Scope] = ["/dir/nested": .shallow, "/dir/nested/file": .entry] + if delivery == "delayed" { + _ = try await observation.refresh(on: fs, signal: .paths(["/dir": .recursive])) + } + var paths = descendants + if delivery == "same batch" { paths["/dir"] = .recursive } + let changes = try await observation.refresh(on: fs, signal: .paths(paths)) + #expect(!changes.changes.contains { $0.path.isWithin("/dir") && $0.path != "/dir" && $0.effect == .created }) + #expect(observation.metadata.entries["/dir/nested"] == nil) + #expect(observation.metadata.entries["/dir/nested/file"] == nil) + if delivery == "delayed" { #expect(changes.changes.isEmpty) } + // With a stale ancestor, its own signal is still needed by the change-source contract. + if delivery == "stale ancestor" { _ = try await observation.refresh(on: fs, signal: .paths(["/dir": .recursive])) } + #expect(try await observation.refresh(on: fs, signal: .rescan).changes.isEmpty) + } + @Test func `shallow parent plus entry signals leave unrelated subtrees and contents alone`() async throws { let fs = SignallingFileSystem() try await fs.base.createDirectory(path: "/unrelated/deep", recursive: true) From 19bb47415c349d5e2de8bcb22402ce21189fa8dd Mon Sep 17 00:00:00 2001 From: Zac White Date: Wed, 21 Oct 2026 11:40:33 -0700 Subject: [PATCH 34/34] Fence FSEvents setup delivery instead of sleeping in the medium-tree watch test The test drained the events from writing its 3,200 fixture files with a fixed 250 ms sleep. On a slow CI runner, late setup batches arrived during the measurement and the refresh read fixture files, failing the only-/x-is-read assertion. FSEvents delivers in order, so the test now writes a marker and waits for its event before measuring, and the assertion names any unexpected reads. --- Tests/WorkspaceTests/ScopedObservationTests.swift | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Tests/WorkspaceTests/ScopedObservationTests.swift b/Tests/WorkspaceTests/ScopedObservationTests.swift index 2a690de..141071d 100644 --- a/Tests/WorkspaceTests/ScopedObservationTests.swift +++ b/Tests/WorkspaceTests/ScopedObservationTests.swift @@ -65,7 +65,7 @@ struct ScopedObservationTests { guard case let .changes(change)? = await first(events) else { Issue.record("missing external event"); return } #expect(change.touchedPaths == ["/x"]) #expect(await fs.base.listings.dropFirst(lists).allSatisfy { $0 == .root }) - #expect(await fs.base.reads.dropFirst(reads).allSatisfy { $0 == "/x" }) + #expect(await fs.base.reads.dropFirst(reads).filter { $0 != "/x" } == []) // The untouched descendants must remain in the observation index. try await fs.base.remove(path: "/unrelated", recursive: true) fs.signal(.paths([.root: .shallow])) @@ -184,8 +184,14 @@ struct ScopedObservationTests { let fs = SignallingFileSystem(base: local, source: local) let ws = Workspace(fileSystem: fs) let events = await ws.events(.init(includeCheckpoints: false)) - // Drain fixture setup notifications before measuring the external write. - try await Task.sleep(for: .milliseconds(250)) + // Drain fixture setup notifications before measuring: FSEvents delivers in + // order, so once the marker's event arrives, setup batches are done. + try await local.writeFile(path: "/marker", data: Data(), append: false) + var drained = false + while case let .changes(setup)? = await first(events) { + if setup.touchedPaths.contains("/marker") { drained = true; break } + } + try #require(drained, "missing setup delivery fence") let lists = await fs.base.listings.count let stats = await fs.base.statCount let reads = await fs.base.reads.count @@ -200,7 +206,7 @@ struct ScopedObservationTests { #expect(refreshed.allSatisfy { $0 == .root }) #expect(refreshed.count <= 4) #expect(await fs.base.statCount - stats < 20) - #expect(await fs.base.reads.dropFirst(reads).allSatisfy { $0 == "/x" }) + #expect(await fs.base.reads.dropFirst(reads).filter { $0 != "/x" } == []) } #endif