diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift index 4662583a2e..86cc92352f 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift @@ -54,10 +54,30 @@ public protocol LineFoldProvider: AnyObject { /// The provider owns this string because it is the only part of the folding system the host application supplies, /// so it is the only part that can reach the application's localized strings. func foldPlaceholderLabel(for summary: FoldPlaceholderSummary) -> String + + // A provider that finds every fold in one pass returns that pass here. The calculator runs it on a copy of the + // text off the main thread and skips the per-line walk, which has to stay on the main thread. + func documentFoldScanner() -> LineFoldDocumentScanner? } public extension LineFoldProvider { func foldPlaceholderLabel(for summary: FoldPlaceholderSummary) -> String { summary.previewText } + + func documentFoldScanner() -> LineFoldDocumentScanner? { + nil + } +} + +public struct LineFoldProviderFold: Sendable, Equatable { + public let depth: Int + public let range: Range + + public init(depth: Int, range: Range) { + self.depth = depth + self.range = range + } } + +public typealias LineFoldDocumentScanner = @Sendable (String) -> [LineFoldProviderFold] diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift index 8714dd2723..f393c4578d 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift @@ -8,33 +8,50 @@ import AppKit import TableProTextEngine -/// `LineFoldCalculator` receives text edits and rebuilds fold regions asynchronously. -/// -/// This is an actor, all methods and modifications happen in isolation in it's async region. All text requests are -/// marked `@MainActor` for safety. +// Each pass carries the document version it read, so the model can drop a result for text that has since changed. +struct LineFoldPass: Sendable { + let storage: LineFoldStorage + let documentVersion: Int +} + +@MainActor +final class LineFoldDocumentVersion { + private(set) var value = 0 + + func advance() { + value &+= 1 + } +} + actor LineFoldCalculator { weak var foldProvider: LineFoldProvider? weak var controller: TextViewController? - var valueStream: AsyncStream + var valueStream: AsyncStream - private var valueStreamContinuation: AsyncStream.Continuation + private var valueStreamContinuation: AsyncStream.Continuation private var textChangedTask: Task? + private let documentVersion: LineFoldDocumentVersion + + private struct PassStart: Sendable { + let documentRange: NSRange + let sizeLimit: Int + let documentVersion: Int + let scanner: LineFoldDocumentScanner? + let text: String? + } - /// Create a new calculator object that listens to a given stream for text changes. - /// - Parameters: - /// - foldProvider: The object to use to calculate fold regions. - /// - controller: The text controller to use for text and attachment fetching. - /// - textChangedStream: A stream of text changes, received as the document is edited. init( foldProvider: LineFoldProvider, controller: TextViewController, + documentVersion: LineFoldDocumentVersion, textChangedStream: AsyncStream ) { // This could be grabbed from the controller, but Swift 6 doesn't like that (concurrency safety) self.foldProvider = foldProvider self.controller = controller - (valueStream, valueStreamContinuation) = AsyncStream.makeStream() + self.documentVersion = documentVersion + (valueStream, valueStreamContinuation) = AsyncStream.makeStream() Task { await listenToTextChanges(textChangedStream: textChangedStream) } } @@ -42,8 +59,6 @@ actor LineFoldCalculator { textChangedTask?.cancel() } - /// Sets up an attached task to listen to values on a stream of text changes. - /// - Parameter textChangedStream: A stream of text changes. private func listenToTextChanges(textChangedStream: AsyncStream) { textChangedTask = Task { for await _ in textChangedStream { @@ -52,18 +67,63 @@ actor LineFoldCalculator { } } - /// Build out the folds for the entire document. - /// - /// For each line in the document, find the indentation level using the ``levelProvider``. At each line, if the - /// indent increases from the previous line, we start a new fold. If it decreases we end the fold we were in. private func buildFoldsForDocument() async { guard let controller = self.controller, let foldProvider = self.foldProvider else { return } - let documentRange = await controller.textView.documentRange - let sizeLimit = await controller.configuration.peripherals.foldingSizeLimit - guard documentRange.length <= sizeLimit else { - valueStreamContinuation.yield(LineFoldStorage(documentLength: documentRange.length)) + let start = await Self.startPass( + controller: controller, + foldProvider: foldProvider, + documentVersion: documentVersion + ) + guard start.documentRange.length <= start.sizeLimit else { + valueStreamContinuation.yield( + LineFoldPass( + storage: LineFoldStorage(documentLength: start.documentRange.length), + documentVersion: start.documentVersion + ) + ) return } + + let folds: [LineFoldStorage.RawFold] + if let scanner = start.scanner, let text = start.text { + folds = scanner(text).map { LineFoldStorage.RawFold(depth: $0.depth, range: $0.range) } + } else { + folds = await walkLines(controller: controller, foldProvider: foldProvider, documentRange: start.documentRange) + } + + await yieldNewStorage( + newFolds: folds, + controller: controller, + documentRange: start.documentRange, + documentVersion: start.documentVersion + ) + } + + // Bridging the storage's mutable string gives an immutable copy, so the scanner can read it from the calculator's + // own executor while the main thread goes on editing. + @MainActor + private static func startPass( + controller: TextViewController, + foldProvider: LineFoldProvider, + documentVersion: LineFoldDocumentVersion + ) -> PassStart { + let documentRange = controller.textView.documentRange + let sizeLimit = controller.configuration.peripherals.foldingSizeLimit + let scanner = documentRange.length <= sizeLimit ? foldProvider.documentFoldScanner() : nil + return PassStart( + documentRange: documentRange, + sizeLimit: sizeLimit, + documentVersion: documentVersion.value, + scanner: scanner, + text: scanner == nil ? nil : controller.textView.textStorage.string + ) + } + + private func walkLines( + controller: TextViewController, + foldProvider: LineFoldProvider, + documentRange: NSRange + ) async -> [LineFoldStorage.RawFold] { var foldCache: [LineFoldStorage.RawFold] = [] // Depth: Open range var openFolds: [Int: LineFoldStorage.RawFold] = [:] @@ -109,19 +169,14 @@ actor LineFoldCalculator { ) ) } - - await yieldNewStorage(newFolds: foldCache, controller: controller, documentRange: documentRange) + return foldCache } - /// Yield a new storage value on the value stream using a new set of folds. - /// - Parameters: - /// - newFolds: The new folds to yield with the storage value. - /// - controller: The text controller used for range and attachment fetching. - /// - documentRange: The total range of the current document. private func yieldNewStorage( newFolds: [LineFoldStorage.RawFold], controller: TextViewController, - documentRange: NSRange + documentRange: NSRange, + documentVersion: Int ) async { let attachments = await controller.textView.layoutManager.attachments .getAttachmentsOverlapping(documentRange) @@ -136,16 +191,17 @@ actor LineFoldCalculator { documentLength: newFolds.max( by: { $0.range.upperBound < $1.range.upperBound } )?.range.upperBound ?? documentRange.length, - folds: newFolds.sorted(by: { $0.range.lowerBound < $1.range.lowerBound }), + // The store keeps the fold set last where two overlap, so of folds that start together the outermost goes + // last and is the one shown, whichever order the provider reported them in. + folds: newFolds.sorted { + $0.range.lowerBound == $1.range.lowerBound ? $0.depth > $1.depth : $0.range.lowerBound < $1.range.lowerBound + }, collapsedRanges: Set(attachments) ) - valueStreamContinuation.yield(storage) + valueStreamContinuation.yield(LineFoldPass(storage: storage, documentVersion: documentVersion)) } - /// Asynchronously gets more line information from the fold provider. - /// Runs on the main thread so all text-related calculations are safe with the main text storage. - /// - /// Has to be an `AsyncSequence` so it can be main actor isolated. + // An `AsyncSequence` so the provider's per-line calls run on the main actor, where the text storage lives. @MainActor struct ChunkedLineIterator: AsyncSequence, AsyncIteratorProtocol { var controller: TextViewController diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift index 6212fa9817..79650179c7 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift @@ -26,24 +26,36 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { @Published var foldCache = LineFoldStorage(documentLength: 0) private var calculator: LineFoldCalculator + // StatementRunController's threshold and delay: below it a pass costs less than a keystroke, above it a pass on + // every key would keep a core busy while the user types. + static let immediateRecalculationLimit = 64_000 + static let recalculationDelay: Duration = .milliseconds(150) + private var textChangedStream: AsyncStream private var textChangedStreamContinuation: AsyncStream.Continuation private var cacheListenTask: Task? + private let documentVersion: LineFoldDocumentVersion + private let clock: any Clock + private var pendingRecalculation: Task? private var pendingRestoreRanges: Set>? private var placeholderTracker: LineFoldPlaceholderTracker? weak var controller: TextViewController? weak var foldView: NSView? - init(controller: TextViewController, foldView: NSView) { + init(controller: TextViewController, foldView: NSView, clock: any Clock = ContinuousClock()) { self.controller = controller self.foldView = foldView + self.clock = clock (textChangedStream, textChangedStreamContinuation) = AsyncStream.makeStream( bufferingPolicy: .bufferingNewest(1) ) + let documentVersion = LineFoldDocumentVersion() + self.documentVersion = documentVersion self.calculator = LineFoldCalculator( foldProvider: controller.foldProvider, controller: controller, + documentVersion: documentVersion, textChangedStream: textChangedStream ) super.init() @@ -51,9 +63,12 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { let calculator = self.calculator cacheListenTask = Task { @MainActor [weak self, weak foldView] in - for await newFolds in await calculator.valueStream { + for await pass in await calculator.valueStream { guard let self else { return } - self.foldCache = newFolds + // An edit after the pass read the text queued another pass; until it lands `storageUpdated` keeps + // the current folds in step with the text. + guard pass.documentVersion == self.documentVersion.value else { continue } + self.foldCache = pass.storage foldView?.needsDisplay = true self.applyPendingRestoreIfPossible() } @@ -78,6 +93,7 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { gutterHover = nil placeholderHover = nil hoveredFold = nil + documentVersion.advance() foldCache = LineFoldStorage(documentLength: controller?.textView.documentRange.length ?? 0) pendingRestoreRanges = nil refresh() @@ -93,6 +109,8 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { placeholderTracker = nil cacheListenTask?.cancel() cacheListenTask = nil + pendingRecalculation?.cancel() + pendingRecalculation = nil } // MARK: - Collapse @@ -256,11 +274,31 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { ) } - /// Recomputes folds for the whole document. Used to catch up after the ribbon has been hidden and is shown again. func refresh() { + pendingRecalculation?.cancel() + pendingRecalculation = nil textChangedStreamContinuation.yield() } + private func scheduleRecalculation(documentLength: Int) { + guard documentLength > Self.immediateRecalculationLimit else { + refresh() + return + } + pendingRecalculation?.cancel() + let clock = clock + pendingRecalculation = Task { [weak self] in + do { + try await clock.sleep(for: Self.recalculationDelay) + } catch { + return + } + guard let self, !Task.isCancelled else { return } + self.pendingRecalculation = nil + self.textChangedStreamContinuation.yield() + } + } + func textStorage( _ textStorage: NSTextStorage, didProcessEditing editedMask: NSTextStorageEditActions, @@ -270,11 +308,12 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { guard editedMask.contains(.editedCharacters) else { return } + documentVersion.advance() foldCache.storageUpdated(editedRange: editedRange, changeInLength: delta) // Recalculating folds walks the whole document. Skip it while the ribbon is hidden; `refresh()` rebuilds when // it is shown again. guard foldView?.isHidden != true else { return } - textChangedStreamContinuation.yield() + scheduleRecalculation(documentLength: textStorage.length) } /// Finds the deepest cached fold and depth of the fold for a line number. diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/LineFoldingTests/LineFoldDocumentScannerTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/LineFoldingTests/LineFoldDocumentScannerTests.swift new file mode 100644 index 0000000000..ac8acd4ed4 --- /dev/null +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/LineFoldingTests/LineFoldDocumentScannerTests.swift @@ -0,0 +1,265 @@ +import AppKit +import Combine +@testable import TableProEditorKit +import TableProTestSupport +import TableProTextEngine +import Testing + +// A provider with a whole-document scanner is folded off the main thread from a copy of the text, so these hold that +// path to the folds the per-line walk builds, and to the document version the model checks before taking a result. +@MainActor +struct LineFoldDocumentScannerTests { + struct Region: Equatable { + let depth: Int + let range: Range + let startLine: Int + let endLine: Int + } + + struct Fold: Equatable { + let depth: Int + let range: Range + } + + final class ScanLog: @unchecked Sendable { + private let lock = NSLock() + private var started = 0 + private var finished = 0 + private var read: [[UInt16]] = [] + let gate: DispatchSemaphore? + + init(gated: Bool = false) { + gate = gated ? DispatchSemaphore(value: 0) : nil + } + + var startedCount: Int { + lock.withLock { started } + } + + var count: Int { + lock.withLock { finished } + } + + var texts: [[UInt16]] { + lock.withLock { read } + } + + func scan(_ text: String) -> [LineFoldProviderFold] { + lock.withLock { started += 1 } + gate?.wait() + // Read after the wait, so a held pass shows whether its copy moved with the edits made meanwhile. + let snapshot = Array(text.utf16) + defer { lock.withLock { finished += 1; read.append(snapshot) } } + return LineFoldDocumentScannerTests.regions(in: text).map { + LineFoldProviderFold(depth: $0.depth, range: $0.range) + } + } + } + + // Braces nest the way a SQL scanner's frames do: a region that opens and closes on one line folds nothing. + nonisolated static func regions(in text: String) -> [Region] { + var stack: [(start: Int, line: Int)] = [] + var line = 0 + var regions: [Region] = [] + for (offset, unit) in text.utf16.enumerated() { + switch unit { + case 0x0A: + line += 1 + case 0x7B: + stack.append((offset + 1, line)) + case 0x7D: + guard let open = stack.popLast(), open.line != line else { continue } + regions.append( + Region(depth: stack.count + 1, range: open.start.. LineFoldDocumentScanner? { + guard offersScanner else { return nil } + let log = log + return { text in log.scan(text) } + } + + func foldLevelAtLine( + lineNumber: Int, + lineRange: NSRange, + previousDepth: Int, + controller: TextViewController + ) -> [LineFoldProviderLineInfo] { + if lineNumber == 0 { + regions = LineFoldDocumentScannerTests.regions(in: controller.textView.string) + } + let ends = regions.filter { $0.endLine == lineNumber }.sorted { $0.depth > $1.depth } + let starts = regions.filter { $0.startLine == lineNumber }.sorted { $0.depth < $1.depth } + return ends.map { .endFold(rangeEnd: $0.range.upperBound, newDepth: $0.depth - 1) } + + starts.map { .startFold(rangeStart: $0.range.lowerBound, newDepth: $0.depth) } + } + } + + // A statement and its parenthesised body both fold from the end of `CREATE TABLE t (`. + @MainActor + final class SharedStartProvider: LineFoldProvider { + let offersScanner: Bool + private let regions = [ + Region(depth: 1, range: 2..<7, startLine: 0, endLine: 2), + Region(depth: 2, range: 2..<6, startLine: 0, endLine: 2), + ] + + init(offersScanner: Bool) { + self.offersScanner = offersScanner + } + + func documentFoldScanner() -> LineFoldDocumentScanner? { + guard offersScanner else { return nil } + let folds = regions.map { LineFoldProviderFold(depth: $0.depth, range: $0.range) } + return { _ in folds } + } + + func foldLevelAtLine( + lineNumber: Int, + lineRange: NSRange, + previousDepth: Int, + controller: TextViewController + ) -> [LineFoldProviderLineInfo] { + let ends = regions.filter { $0.endLine == lineNumber }.sorted { $0.depth > $1.depth } + let starts = regions.filter { $0.startLine == lineNumber }.sorted { $0.depth < $1.depth } + return ends.map { .endFold(rangeEnd: $0.range.upperBound, newDepth: $0.depth - 1) } + + starts.map { .startFold(rangeStart: $0.range.lowerBound, newDepth: $0.depth) } + } + } + + private static let nestedDocument: String = (0..<40).map { index in + switch index % 4 { + case 0: return "block \(index) {\n inner {\n x\n }\n y\n}" + case 1: return "flat \(index) { same line }" + case 2: return "deep {\n {\n {\n z\n }\n }\n}" + default: return "open {\n a\n} close { b\n}" + } + }.joined(separator: "\n") + + private func makeController(_ text: String, provider: LineFoldProvider) -> TextViewController { + let controller = Mock.textViewController(theme: Mock.theme()) + controller.textView.string = text + controller.textView.frame = NSRect(x: 0, y: 0, width: 1_000, height: 1_000) + controller.textView.updatedViewport(NSRect(x: 0, y: 0, width: 1_000, height: 1_000)) + controller.foldProvider = provider + return controller + } + + private func folds(of model: LineFoldModel, in controller: TextViewController) -> [Fold] { + model.getFolds(in: 0.. [Fold] { + Self.regions(in: text) + .map { Fold(depth: $0.depth, range: $0.range) } + .sorted { $0.range.lowerBound < $1.range.lowerBound } + } + + @Test("The scanner path builds the same folds as the per-line walk") + func scannerMatchesLineWalk() async throws { + let walking = makeController(Self.nestedDocument, provider: BraceFoldProvider(offersScanner: false)) + let scanning = makeController(Self.nestedDocument, provider: BraceFoldProvider(offersScanner: true)) + let walkModel = LineFoldModel(controller: walking, foldView: NSView()) + let scanModel = LineFoldModel(controller: scanning, foldView: NSView()) + + try await waitUntil(timeout: .seconds(60)) { + !folds(of: walkModel, in: walking).isEmpty && !folds(of: scanModel, in: scanning).isEmpty + } + + let walked = folds(of: walkModel, in: walking) + #expect(walked.count > 20) + #expect(folds(of: scanModel, in: scanning) == walked) + #expect(walked == expected(Self.nestedDocument)) + } + + @Test("Of folds that start together, both paths show the outer one") + func sharedStartShowsTheOuterFold() async throws { + let text = "a(\n b\n);\n" + let walking = makeController(text, provider: SharedStartProvider(offersScanner: false)) + let scanning = makeController(text, provider: SharedStartProvider(offersScanner: true)) + let walkModel = LineFoldModel(controller: walking, foldView: NSView()) + let scanModel = LineFoldModel(controller: scanning, foldView: NSView()) + + try await waitUntil(timeout: .seconds(30)) { + !folds(of: walkModel, in: walking).isEmpty && !folds(of: scanModel, in: scanning).isEmpty + } + + #expect(folds(of: walkModel, in: walking) == [Fold(depth: 1, range: 2..<7)]) + #expect(folds(of: scanModel, in: scanning) == [Fold(depth: 1, range: 2..<7)]) + } + + @Test("A pass over text that changed before it landed is dropped") + func stalePassIsDropped() async throws { + let log = ScanLog(gated: true) + // A failed wait must not leave a pool thread parked in the scanner for the rest of the run. + defer { + log.gate?.signal() + log.gate?.signal() + } + let original = "a {\n b\n}\nc {\n d\n}\n" + let controller = makeController(original, provider: BraceFoldProvider(offersScanner: true, log: log)) + let model = LineFoldModel(controller: controller, foldView: NSView()) + + var published: [[Fold]] = [] + let subscription = model.$foldCache.sink { cache in + published.append(cache.folds(in: 0.. LineFoldModel.immediateRecalculationLimit) + let controller = makeController(large, provider: BraceFoldProvider(offersScanner: true, log: log)) + let model = LineFoldModel(controller: controller, foldView: NSView(), clock: clock) + try await waitUntil(timeout: .seconds(20)) { log.count == 1 && !folds(of: model, in: controller).isEmpty } + + controller.textView.textStorage.replaceCharacters(in: NSRange(location: 0, length: 0), with: "z") + await clock.waitUntilSleeping() + #expect(log.count == 1) + + clock.advance(by: LineFoldModel.recalculationDelay) + try await waitUntil(timeout: .seconds(20)) { log.count == 2 } + + let smallLog = ScanLog() + let small = makeController("x {\n y\n}\n", provider: BraceFoldProvider(offersScanner: true, log: smallLog)) + let smallModel = LineFoldModel(controller: small, foldView: NSView(), clock: clock) + try await waitUntil(timeout: .seconds(20)) { smallLog.count == 1 } + small.textView.textStorage.replaceCharacters(in: NSRange(location: 0, length: 0), with: "z") + try await waitUntil(timeout: .seconds(20)) { smallLog.count == 2 } + _ = smallModel + } +} diff --git a/apple/TablePro/Core/AI/Copilot/CopilotDocumentServer.swift b/apple/TablePro/Core/AI/Copilot/CopilotDocumentServer.swift new file mode 100644 index 0000000000..da0fb9b029 --- /dev/null +++ b/apple/TablePro/Core/AI/Copilot/CopilotDocumentServer.swift @@ -0,0 +1,38 @@ +import Foundation + +@MainActor +protocol CopilotDocumentServer { + var generation: Int { get } + var hasClient: Bool { get } + func didOpen(_ document: LSPTextDocumentItem) async + func didChange(_ change: LSPDidChangeParams) async + func didFocus(uri: String) async + func didClose(uri: String) async +} + +@MainActor +struct LiveCopilotDocumentServer: CopilotDocumentServer { + var generation: Int { CopilotService.shared.generation } + + var hasClient: Bool { CopilotService.shared.client != nil } + + func didOpen(_ document: LSPTextDocumentItem) async { + await CopilotService.shared.client?.didOpenDocument(document) + } + + func didChange(_ change: LSPDidChangeParams) async { + await CopilotService.shared.client?.didChangeDocument( + uri: change.textDocument.uri, + version: change.textDocument.version, + changes: change.contentChanges + ) + } + + func didFocus(uri: String) async { + await CopilotService.shared.client?.didFocusDocument(uri: uri) + } + + func didClose(uri: String) async { + await CopilotService.shared.client?.didCloseDocument(uri: uri) + } +} diff --git a/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift b/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift index 20a137cb63..9b2d594d07 100644 --- a/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift +++ b/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift @@ -1,117 +1,384 @@ -// -// CopilotDocumentSync.swift -// TablePro -// - -import Foundation +import AppKit import os +import TableProTextEngine -/// Manages LSP document lifecycle for Copilot. Prepends the schema preamble -/// to all document text sent to the server. @MainActor final class CopilotDocumentSync { + private enum Outgoing { + case open(uri: String, text: String, generation: Int) + case change(uri: String, changes: [LSPTextDocumentContentChangeEvent], generation: Int) + case focus(uri: String, generation: Int) + case close(uri: String, generation: Int) + + var generation: Int { + switch self { + case .open(_, _, let generation), .change(_, _, let generation), + .focus(_, let generation), .close(_, let generation): + generation + } + } + } + + private struct OpenDocument { + let uri: String + let generation: Int + } + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "CopilotDocumentSync") + private static let carriageReturn: unichar = 0x0D + private static let lineFeed: unichar = 0x0A - private let documentManager = LSPDocumentManager() let preambleBuilder = CopilotPreambleBuilder() - private var currentURI: String? - private var serverSyncedURIs: Set = [] - private var pendingText: [String: String] = [:] + private let server: any CopilotDocumentServer + private let lengthLimit: Int + private let documentManager = LSPDocumentManager() + // Every editor talks to the one server, so a document number is never reused by another editor's sync. + private static var nextDocumentNumber = 1 private var uriMap: [UUID: String] = [:] - private var nextID = 1 - private var lastKnownGeneration: Int = 0 + + private var openDocument: OpenDocument? + private var needsFullText = false + private var mayContainCarriageReturn = false + private var linesMayDisagree = false + private var lineCheckTask: Task? + + private var outbox: [Outgoing] = [] + private var drainTask: Task? + + private weak var textView: TextView? + private weak var observedStorage: NSTextStorage? + private var editObservation: CopilotEditObservation? + private var currentTabID: @MainActor () -> UUID? = { nil } + private var isAllowed: @MainActor () -> Bool = { false } + private var schemaSource: @MainActor () -> CopilotPreambleBuilder.Source? = { nil } + private var preambleRebuild: Task? + + init(server: any CopilotDocumentServer, lengthLimit: Int) { + self.server = server + self.lengthLimit = lengthLimit + } func documentURI(for tabID: UUID) -> String { if let existing = uriMap[tabID] { return existing } - let fileURL = CopilotPreambleBuilder.contextDirectory.appendingPathComponent("query-\(nextID).sql") - nextID += 1 + let fileURL = CopilotPreambleBuilder.contextDirectory + .appendingPathComponent("query-\(Self.nextDocumentNumber).sql") + Self.nextDocumentNumber += 1 let uri = fileURL.absoluteString uriMap[tabID] = uri return uri } - func resetServerState() { - serverSyncedURIs.removeAll() - documentManager.resetAll() - pendingText.removeAll() + // MARK: - Editor + + func attach( + to textView: TextView, + tabID: @escaping @MainActor () -> UUID?, + isAllowed: @escaping @MainActor () -> Bool, + schemaSource: @escaping @MainActor () -> CopilotPreambleBuilder.Source? + ) { + self.textView = textView + currentTabID = tabID + self.isAllowed = isAllowed + self.schemaSource = schemaSource + guard let storage = textView.textStorage, let lines = textView.layoutManager?.lineStorage else { return } + observe(storage) + // Edits made before this observer existed can have left the line storage off the server's lines. + if storage.mutableString.range(of: "\r", options: .literal).location != NSNotFound { + linesMayDisagree = !lines.breaksLines(like: storage.mutableString) + } + synchronize(from: storage) + } + + // Only a tab the user types in, or asks a suggestion in, reaches the server. + func textViewDidReplaceDocument() { + guard let storage = textView?.textStorage else { return } + observe(storage) + needsFullText = true + } + + func detach() { + preambleRebuild?.cancel() + preambleRebuild = nil + editObservation = nil + observedStorage = nil + textView = nil + currentTabID = { nil } + isAllowed = { false } + schemaSource = { nil } + guard let open = openDocument else { return } + openDocument = nil + enqueue(.close(uri: open.uri, generation: open.generation)) } - /// Register document locally. Does NOT send to server. - func ensureDocumentOpen(tabID: UUID, text: String, languageId: String = "sql") { + func activate(tabID: UUID, text: String) { + guard canSend(length: text.utf16.count) else { + needsFullText = true + return + } let uri = documentURI(for: tabID) - let fullText = preambleBuilder.prependToText(text) - if !documentManager.isOpen(uri) { - _ = documentManager.openDocument(uri: uri, languageId: languageId, text: fullText) + let generation = server.generation + if let open = openDocument, open.uri != uri { + enqueue(.close(uri: open.uri, generation: open.generation)) + openDocument = nil } - currentURI = uri + sendFullText(text, uri: uri, generation: generation) + enqueue(.focus(uri: uri, generation: generation)) } - /// Open document at the server with preamble-prepended text - func didActivateTab(tabID: UUID, text: String, languageId: String = "sql") async { - let currentGeneration = CopilotService.shared.generation - if currentGeneration != lastKnownGeneration { - resetServerState() - lastKnownGeneration = currentGeneration + // MARK: - Server + + // Column reads can queue behind a running query, so a request never waits on them; the next sync sends the result. + func refreshPreamble(forStatementAt cursorOffset: Int, in text: String) { + guard preambleRebuild == nil, textView != nil, isAllowed(), let source = schemaSource() else { return } + preambleRebuild = Task { [weak self] in + guard let self else { return } + let rebuilt = await self.preambleBuilder.rebuiltPreamble(forStatementAt: cursorOffset, in: text, from: source) + self.preambleRebuild = nil + guard !Task.isCancelled, let rebuilt else { return } + let changesText = rebuilt.preamble != self.preambleBuilder.preamble + guard self.preambleBuilder.adopt(rebuilt), changesText else { return } + // Set with the new preamble, so no edit is placed with its line count against a document that has the old one. + self.needsFullText = true + } + } + + func flush() async { + if lineCheckTask != nil { + checkLines() + } + resynchronizeIfStale() + while let task = drainTask { + await task.value + } + } + + func currentDocumentInfo() -> (uri: String, version: Int)? { + guard let open = openDocument, !needsFullText, outbox.isEmpty, drainTask == nil, + open.generation == server.generation, + let tabID = currentTabID(), uriMap[tabID] == open.uri, + let version = documentManager.version(for: open.uri) else { return nil } + return (open.uri, version) + } + + // MARK: - Edits + + private func observe(_ storage: NSTextStorage) { + observedStorage = storage + linesMayDisagree = false + // Every way the text changes passes through the storage, the typing filters' edits included. + let token = NotificationCenter.default.addObserver( + forName: NSTextStorage.willProcessEditingNotification, + object: storage, + queue: nil + ) { [weak self] _ in + MainActor.assumeIsolated { self?.storageWillProcessEditing() } } + editObservation = CopilotEditObservation(token) + } + private func storageWillProcessEditing() { + guard let storage = observedStorage, storage.editedMask.contains(.editedCharacters) else { return } + guard textView?.textStorage === storage else { + needsFullText = true + return + } + guard let tabID = currentTabID(), canSend(length: storage.length) else { + needsFullText = true + return + } let uri = documentURI(for: tabID) - let fullText = preambleBuilder.prependToText(text) - ensureDocumentOpen(tabID: tabID, text: text, languageId: languageId) - - guard let client = CopilotService.shared.client else { return } - if !serverSyncedURIs.contains(uri) { - let item = LSPTextDocumentItem( - uri: uri, - languageId: languageId, - version: documentManager.version(for: uri) ?? 0, - text: fullText - ) - await client.didOpenDocument(item) - serverSyncedURIs.insert(uri) - - if let pending = pendingText.removeValue(forKey: uri) { - let pendingFull = preambleBuilder.prependToText(pending) - if let (versioned, changes) = documentManager.changeDocument(uri: uri, newText: pendingFull) { - await client.didChangeDocument(uri: versioned.uri, version: versioned.version, changes: changes) - } + guard let open = openDocument, open.uri == uri, open.generation == server.generation else { + activate(tabID: tabID, text: storage.string) + return + } + guard !needsFullText, let change = incrementalChange(in: storage) else { + sendFullText(storage.string, uri: uri, generation: open.generation) + return + } + enqueueChange(change, uri: uri, generation: open.generation) + } + + private func incrementalChange(in storage: NSTextStorage) -> LSPTextDocumentContentChangeEvent? { + guard let lines = textView?.layoutManager?.lineStorage else { return nil } + let editedRange = storage.editedRange + let replacedLength = editedRange.length - storage.changeInLength + // The layout manager updates its lines after this notification, so they still describe the text before the edit. + guard lines.length == storage.length - storage.changeInLength else { + Self.logger.debug("Line storage out of step with the edit, sending the full text") + return nil + } + let text = storage.mutableString + let inserted = text.substring(with: editedRange) + if inserted.utf16.contains(Self.carriageReturn) { + mayContainCarriageReturn = true + } + if mayContainCarriageReturn { + if Self.touchesLineBreakPair(in: text, editedRange: editedRange, replacedLength: replacedLength) { + linesMayDisagree = true } + if linesMayDisagree { return nil } + } + let lineOffset = preambleBuilder.preambleLineCount + guard let start = Self.position(of: editedRange.location, in: lines, lineOffset: lineOffset), + let end = Self.position(of: editedRange.location + replacedLength, in: lines, lineOffset: lineOffset) else { + return nil } - await client.didFocusDocument(uri: uri) + return LSPTextDocumentContentChangeEvent(range: LSPRange(start: start, end: end), text: inserted) } - /// Send text change with preamble prepended - func didChangeText(tabID: UUID, newText: String) async { - let currentGeneration = CopilotService.shared.generation - if currentGeneration != lastKnownGeneration { - resetServerState() - lastKnownGeneration = currentGeneration + // The server clamps a position between CR and LF to before the CR, and the editor's line storage splits or joins a + // CR LF pair an edit crosses, so such an edit sends the full text, as does every later one until no CR is left. + private static func touchesLineBreakPair(in text: NSString, editedRange: NSRange, replacedLength: Int) -> Bool { + if editedRange.location > 0, text.character(at: editedRange.location - 1) == carriageReturn { + return true } + let end = NSMaxRange(editedRange) + guard end < text.length, text.character(at: end) == lineFeed else { return false } + return replacedLength > 0 || (editedRange.length > 0 && text.character(at: end - 1) == carriageReturn) + } + + private static func position(of offset: Int, in lines: TextLineStorage, lineOffset: Int) -> LSPPosition? { + guard let line = offset == lines.length ? lines.last : lines.getLine(atOffset: offset) else { return nil } + return LSPPosition(line: line.index + lineOffset, character: offset - line.range.location) + } + // MARK: - Full Text + + private func canSend(length: Int) -> Bool { + length <= lengthLimit && server.hasClient && isAllowed() + } + + private func synchronize(from storage: NSTextStorage) { + guard let tabID = currentTabID(), canSend(length: storage.length) else { + needsFullText = true + return + } let uri = documentURI(for: tabID) - guard serverSyncedURIs.contains(uri) else { - pendingText[uri] = newText + if let open = openDocument, open.uri == uri, open.generation == server.generation { + sendFullText(storage.string, uri: uri, generation: open.generation) + } else { + activate(tabID: tabID, text: storage.string) + } + } + + private func resynchronizeIfStale() { + guard let storage = textView?.textStorage else { return } + if storage !== observedStorage { + observe(storage) + needsFullText = true + } + guard let tabID = currentTabID() else { return } + if let open = openDocument, open.uri == uriMap[tabID], open.generation == server.generation, !needsFullText { return } - let fullText = preambleBuilder.prependToText(newText) - guard let client = CopilotService.shared.client else { return } - guard let (versioned, changes) = documentManager.changeDocument(uri: uri, newText: fullText) else { return } - await client.didChangeDocument(uri: versioned.uri, version: versioned.version, changes: changes) + synchronize(from: storage) } - func didCloseTab(tabID: UUID) async { - let uri = documentURI(for: tabID) - guard let client = CopilotService.shared.client else { return } - guard let docId = documentManager.closeDocument(uri: uri) else { return } - await client.didCloseDocument(uri: docId.uri) - serverSyncedURIs.remove(uri) - uriMap.removeValue(forKey: tabID) - if currentURI == uri { currentURI = nil } + private func sendFullText(_ text: String, uri: String, generation: Int) { + mayContainCarriageReturn = (text as NSString).range(of: "\r", options: .literal).location != NSNotFound + if !mayContainCarriageReturn { + linesMayDisagree = false + } else if linesMayDisagree { + scheduleLineCheck() + } + needsFullText = false + let fullText = preambleBuilder.prependToText(text) + if let open = openDocument, open.uri == uri, open.generation == generation { + enqueueChange(LSPTextDocumentContentChangeEvent(text: fullText), uri: uri, generation: generation) + } else { + openDocument = OpenDocument(uri: uri, generation: generation) + enqueue(.open(uri: uri, text: fullText, generation: generation)) + } } - func currentDocumentInfo() -> (uri: String, version: Int)? { - guard let uri = currentURI else { return nil } - guard serverSyncedURIs.contains(uri) else { return nil } - guard let version = documentManager.version(for: uri) else { return nil } - return (uri, version) + // The storage notifies before the layout manager takes an edit in, so the check runs once the edit is done. + private func scheduleLineCheck() { + guard lineCheckTask == nil else { return } + lineCheckTask = Task { [weak self] in + guard !Task.isCancelled else { return } + self?.checkLines() + } + } + + // Every edit since the last full text went out as full text, so lines that follow the text now follow the server's. + private func checkLines() { + lineCheckTask?.cancel() + lineCheckTask = nil + guard linesMayDisagree, let textView, let storage = textView.textStorage, storage === observedStorage, + storage.editedMask.isEmpty, let lines = textView.layoutManager?.lineStorage else { return } + linesMayDisagree = !lines.breaksLines(like: storage.mutableString) + } + + // MARK: - Outbox + + private func enqueue(_ message: Outgoing) { + outbox.append(message) + scheduleDrain() + } + + private func enqueueChange(_ change: LSPTextDocumentContentChangeEvent, uri: String, generation: Int) { + if change.range == nil { + outbox.removeAll { message in + if case .change(let queuedURI, _, _) = message { return queuedURI == uri } + return false + } + } + if case .change(let lastURI, let changes, let lastGeneration)? = outbox.last, + lastURI == uri, lastGeneration == generation { + outbox[outbox.count - 1] = .change(uri: uri, changes: changes + [change], generation: generation) + return + } + enqueue(.change(uri: uri, changes: [change], generation: generation)) + } + + // One task sends everything in edit order; a task per notification gives no order between them. + private func scheduleDrain() { + guard drainTask == nil else { return } + drainTask = Task { + await self.drainOutbox() + } + } + + private func drainOutbox() async { + while !outbox.isEmpty { + await send(outbox.removeFirst()) + } + drainTask = nil + } + + private func send(_ message: Outgoing) async { + guard server.hasClient, server.generation == message.generation else { return } + switch message { + case let .close(uri, _): + guard documentManager.closeDocument(uri: uri) != nil else { return } + await server.didClose(uri: uri) + case let .open(uri, _, generation) where !isAllowed(): + if openDocument?.uri == uri, openDocument?.generation == generation { + openDocument = nil + } + case .change where !isAllowed(): + needsFullText = true + case .focus where !isAllowed(): + break + case let .open(uri, text, _): + await server.didOpen(documentManager.openDocument(uri: uri, languageId: "sql", text: text)) + case let .change(uri, changes, _): + guard let params = documentManager.changeDocument(uri: uri, changes: changes) else { return } + await server.didChange(params) + case let .focus(uri, _): + await server.didFocus(uri: uri) + } + } +} + +private final class CopilotEditObservation { + private let token: any NSObjectProtocol + + init(_ token: any NSObjectProtocol) { + self.token = token + } + + deinit { + NotificationCenter.default.removeObserver(token) } } diff --git a/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift b/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift index 6c5f4a1f8e..2733c6abf7 100644 --- a/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift +++ b/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift @@ -22,7 +22,10 @@ final class CopilotInlineSource: InlineSuggestionSource { } func requestSuggestion(context: SuggestionContext) async throws -> InlineSuggestion? { - guard let client = CopilotService.shared.client else { return nil } + guard CopilotService.shared.client != nil else { return nil } + documentSync.refreshPreamble(forStatementAt: context.cursorOffset, in: context.fullText) + await documentSync.flush() + guard !Task.isCancelled, let client = CopilotService.shared.client else { return nil } guard let docInfo = documentSync.currentDocumentInfo() else { return nil } let editorSettings = AppSettingsManager.shared.editor diff --git a/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift b/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift index 4ba91198ca..514cc3a350 100644 --- a/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift +++ b/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift @@ -6,9 +6,36 @@ import Foundation import os import TableProPluginKit +import TableProSQLGrammar @MainActor final class CopilotPreambleBuilder { + struct Source { + let schemaProvider: SQLSchemaProvider + let databaseName: String + let databaseType: DatabaseType + let grammar: SQLLexicalGrammar + let budget: Int + } + + struct Rebuilt { + let preamble: String + let lineCount: Int + fileprivate let describedTables: [TableKey] + fileprivate let inputs: Source + fileprivate let revision: Int + } + + fileprivate struct TableKey: Hashable { + let name: String + let schema: String? + + init(_ table: TableInfo) { + name = table.name + schema = table.schema + } + } + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "CopilotPreambleBuilder") static let contextDirectory: URL = { @@ -20,33 +47,147 @@ final class CopilotPreambleBuilder { private(set) var preambleLineCount: Int = 0 + private var inputs: Source? + // Every table whose columns were read, with or without any, so a table that has none is not read on every request. + private var describedTables: [TableKey] = [] + private var revision = 0 + func buildPreamble( schemaProvider: SQLSchemaProvider, databaseName: String, - databaseType: DatabaseType + databaseType: DatabaseType, + referencing documentText: String? = nil, + grammar: SQLLexicalGrammar? = nil, + maxTables: Int = AppSettingsManager.shared.ai.maxSchemaTables ) async { try? FileManager.default.createDirectory(at: Self.contextDirectory, withIntermediateDirectories: true) + let inputs = Source( + schemaProvider: schemaProvider, + databaseName: databaseName, + databaseType: databaseType, + grammar: grammar ?? databaseType.lexicalGrammar, + budget: max(maxTables, 0) + ) + self.inputs = inputs let tables = await schemaProvider.getTables() - guard !tables.isEmpty else { - preamble = "" - preambleLineCount = 0 - return + let defaultSchema = await schemaProvider.getDefaultSchema() + var first: [Int] = [] + if tables.count > inputs.budget, let documentText { + first = await Self.referencedTableIndices( + in: documentText, + cursorOffset: nil, + grammar: inputs.grammar, + tables: tables, + defaultSchema: defaultSchema + ) } + let picked = Self.budgeted(tables, first: first, keeping: [], budget: inputs.budget) + guard let rendered = await Self.render( + picked, catalogCount: tables.count, inputs: inputs, defaultSchema: defaultSchema + ) else { return } + apply(rendered) - let defaultSchema = await schemaProvider.getDefaultSchema() + Self.logger.info("Copilot schema preamble: \(tables.count) tables, \(self.preambleLineCount) lines") + } + + // A tab moved to another database scope names that scope's tables, so a new scope starts the preamble over, as + // does an editor that had no schema when Copilot started. + func rebuiltPreamble(forStatementAt cursorOffset: Int, in text: String, from source: Source) async -> Rebuilt? { + let base = revision + // A lower "Max schema tables" has to take effect on the next request, as it does in the chat. + let startsOver = inputs.map { + $0.schemaProvider !== source.schemaProvider || $0.databaseName != source.databaseName + || $0.budget != source.budget || $0.databaseType != source.databaseType || $0.grammar != source.grammar + } ?? true + let target = startsOver ? source : inputs ?? source + let kept = startsOver ? [] : describedTables + let described = Set(kept) + let tables = await target.schemaProvider.getTables() + if !startsOver, tables.count <= target.budget, tables.allSatisfy({ described.contains(TableKey($0)) }) { + return nil + } + let defaultSchema = await target.schemaProvider.getDefaultSchema() + let referenced = await Self.referencedTableIndices( + in: text, + cursorOffset: cursorOffset, + grammar: target.grammar, + tables: tables, + defaultSchema: defaultSchema + ).prefix(target.budget) + guard startsOver || referenced.contains(where: { !described.contains(TableKey(tables[$0])) }) else { return nil } + let picked = Self.budgeted(tables, first: Array(referenced), keeping: kept, budget: target.budget) + guard let rendered = await Self.render( + picked, catalogCount: tables.count, inputs: target, defaultSchema: defaultSchema + ) else { return nil } + return Rebuilt( + preamble: rendered.preamble, + lineCount: rendered.lineCount, + describedTables: rendered.describedTables, + inputs: target, + revision: base + ) + } + + // A rebuild that started before another build landed describes a preamble the sync no longer has. + func adopt(_ rebuilt: Rebuilt) -> Bool { + guard rebuilt.revision == revision else { return false } + apply(rebuilt) + return true + } + func prependToText(_ text: String) -> String { + guard !preamble.isEmpty else { return text } + return preamble + text + } + + private func apply(_ rebuilt: Rebuilt) { + preamble = rebuilt.preamble + preambleLineCount = rebuilt.lineCount + describedTables = rebuilt.describedTables + inputs = rebuilt.inputs + revision += 1 + } + + private static func budgeted(_ tables: [TableInfo], first: [Int], keeping kept: [TableKey], budget: Int) -> [TableInfo] { + var catalog: [TableKey: TableInfo] = [:] + for table in tables where catalog[TableKey(table)] == nil { + catalog[TableKey(table)] = table + } + var picked: [TableInfo] = [] + var seen = Set() + for table in first.map({ tables[$0] }) + kept.compactMap({ catalog[$0] }) + tables { + guard picked.count < budget else { break } + if seen.insert(TableKey(table)).inserted { + picked.append(table) + } + } + return picked + } + + private static func render( + _ picked: [TableInfo], + catalogCount: Int, + inputs: Source, + defaultSchema: String? + ) async -> Rebuilt? { + guard catalogCount > 0 else { + return Rebuilt(preamble: "", lineCount: 0, describedTables: [], inputs: inputs, revision: 0) + } + + let dialect = singleLine(inputs.databaseType.rawValue) var lines: [String] = [] - lines.append("-- Database: \(databaseName)") - lines.append("-- Dialect: \(databaseType.rawValue): use \(databaseType.rawValue) syntax only") + lines.append("-- Database: \(singleLine(inputs.databaseName))") + lines.append("-- Dialect: \(dialect): use \(dialect) syntax only") lines.append("--") - for table in tables { - let columns = await schemaProvider.getColumns(for: table.name, schema: table.schema) + for table in picked { + guard !Task.isCancelled else { return nil } + let columns = await inputs.schemaProvider.getColumns(for: table.name, schema: table.schema) guard !columns.isEmpty else { continue } let colDefs = columns.map { col -> String in - var parts = ["\(col.name) \(col.dataType)"] + var parts = ["\(singleLine(col.name)) \(singleLine(col.dataType))"] if col.isPrimaryKey { parts.append("PK") } if !col.isNullable { parts.append("NOT NULL") } return parts.joined(separator: " ") @@ -57,19 +198,45 @@ final class CopilotPreambleBuilder { implicitSchemaName: defaultSchema, quote: { $0 } ) - lines.append("-- \(tableName)(\(colDefs.joined(separator: ", ")))") + lines.append("-- \(singleLine(tableName))(\(colDefs.joined(separator: ", ")))") } - lines.append("") + if catalogCount > picked.count { + lines.append("-- … and \(catalogCount - picked.count) more tables") + } - preamble = lines.joined(separator: "\n") - preambleLineCount = lines.count - 1 + lines.append("") + return Rebuilt( + preamble: lines.joined(separator: "\n"), + lineCount: lines.count - 1, + describedTables: picked.map(TableKey.init), + inputs: inputs, + revision: 0 + ) + } - Self.logger.info("Copilot schema preamble: \(tables.count) tables, \(self.preambleLineCount) lines") + // The document sync places edits by line number, so a name must not add a line break. + private static func singleLine(_ name: String) -> String { + name.replacingOccurrences(of: "\r", with: " ", options: .literal) + .replacingOccurrences(of: "\n", with: " ", options: .literal) } - func prependToText(_ text: String) -> String { - guard !preamble.isEmpty else { return text } - return preamble + text + // Scanning a large script takes too long for the main actor. + @concurrent + nonisolated private static func referencedTableIndices( + in text: String, + cursorOffset: Int?, + grammar: SQLLexicalGrammar, + tables: [TableInfo], + defaultSchema: String? + ) async -> [Int] { + let scope = cursorOffset.map { + SQLStatementScanner.statementAtCursor(in: text, cursorPosition: $0, grammar: grammar) + } ?? text + return AISchemaContext.indices( + of: AISchemaContext.referencedTables(in: scope, grammar: grammar), + in: tables, + defaultSchema: defaultSchema + ) } } diff --git a/apple/TablePro/Core/LSP/LSPDocumentManager.swift b/apple/TablePro/Core/LSP/LSPDocumentManager.swift index 787d684eb0..cd56839cd9 100644 --- a/apple/TablePro/Core/LSP/LSPDocumentManager.swift +++ b/apple/TablePro/Core/LSP/LSPDocumentManager.swift @@ -7,47 +7,28 @@ import Foundation @MainActor final class LSPDocumentManager { - struct DocumentState { - var uri: String - var version: Int - var languageId: String - } - - private var documents: [String: DocumentState] = [:] + private var versions: [String: Int] = [:] func openDocument(uri: String, languageId: String, text: String) -> LSPTextDocumentItem { - let state = DocumentState(uri: uri, version: 0, languageId: languageId) - documents[uri] = state + versions[uri] = 0 return LSPTextDocumentItem(uri: uri, languageId: languageId, version: 0, text: text) } - func changeDocument( - uri: String, - newText: String - ) -> (versioned: LSPVersionedTextDocumentIdentifier, changes: [LSPTextDocumentContentChangeEvent])? { - guard var state = documents[uri] else { return nil } - state.version += 1 - documents[uri] = state - return ( - versioned: LSPVersionedTextDocumentIdentifier(uri: uri, version: state.version), - changes: [LSPTextDocumentContentChangeEvent(text: newText)] + func changeDocument(uri: String, changes: [LSPTextDocumentContentChangeEvent]) -> LSPDidChangeParams? { + guard let version = versions[uri] else { return nil } + versions[uri] = version + 1 + return LSPDidChangeParams( + textDocument: LSPVersionedTextDocumentIdentifier(uri: uri, version: version + 1), + contentChanges: changes ) } func closeDocument(uri: String) -> LSPTextDocumentIdentifier? { - guard documents.removeValue(forKey: uri) != nil else { return nil } + guard versions.removeValue(forKey: uri) != nil else { return nil } return LSPTextDocumentIdentifier(uri: uri) } func version(for uri: String) -> Int? { - documents[uri]?.version - } - - func isOpen(_ uri: String) -> Bool { - documents[uri] != nil - } - - func resetAll() { - documents.removeAll() + versions[uri] } } diff --git a/apple/TablePro/Core/LSP/LSPTypes.swift b/apple/TablePro/Core/LSP/LSPTypes.swift index 0df7dd49b7..f6888062ad 100644 --- a/apple/TablePro/Core/LSP/LSPTypes.swift +++ b/apple/TablePro/Core/LSP/LSPTypes.swift @@ -35,7 +35,18 @@ struct LSPVersionedTextDocumentIdentifier: Codable, Sendable, Equatable { } struct LSPTextDocumentContentChangeEvent: Codable, Sendable, Equatable { + let range: LSPRange? let text: String + + init(text: String) { + self.range = nil + self.text = text + } + + init(range: LSPRange, text: String) { + self.range = range + self.text = text + } } struct LSPInlineCompletionItem: Codable, Sendable, Equatable { diff --git a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift index 9d8bde2d77..fb47db43f7 100644 --- a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift +++ b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift @@ -1,151 +1,116 @@ -// -// VimTextBufferAdapter.swift -// TablePro -// -// Adapts TableProTextEngine's TextView to the VimTextBuffer protocol -// - import AppKit import Foundation import TableProTextEngine -/// Bridges TableProTextEngine's TextView to VimTextBuffer for the Vim engine @MainActor final class VimTextBufferAdapter: VimTextBuffer { private weak var textView: TextView? + private struct LineStorageTrust { + weak var storage: NSTextStorage? + let length: Int + let trusted: Bool + } + + private var lineStorageTrust: LineStorageTrust? + init(textView: TextView) { self.textView = textView } - private var cachedLineCount: Int? + private var text: NSString? { + textView?.textStorage?.mutableString + } // MARK: - VimTextBuffer var length: Int { - guard let textView else { return 0 } - return (textView.string as NSString).length + textView?.textStorage?.length ?? 0 } var lineCount: Int { - if let cached = cachedLineCount { return cached } - guard let textView else { return 1 } - let nsString = textView.string as NSString - if nsString.length == 0 { - cachedLineCount = 1 - return 1 + guard let text, text.length > 0 else { return 1 } + let endsWithLineBreak = isLineBreak(text.character(at: text.length - 1)) + // The line storage keeps an empty line after a final line ending, which Vim does not count. + if let lines = lineStorage(matching: text), let last = lines.last, (last.range.length == 0) == endsWithLineBreak { + return max(1, endsWithLineBreak ? lines.count - 1 : lines.count) } var count = 0 - var index = 0 - while index < nsString.length { - let lineRange = nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) + var location = 0 + while location < text.length { + location = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0))) count += 1 - index = lineRange.location + lineRange.length } - let result = max(1, count) - cachedLineCount = result - return result + return max(1, count) } func invalidateLineCache() { - cachedLineCount = nil + lineStorageTrust = nil } func lineRange(forOffset offset: Int) -> NSRange { - guard let textView else { return NSRange(location: 0, length: 0) } - let nsString = textView.string as NSString - let clampedOffset = min(max(0, offset), nsString.length) - return nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: clampedOffset, length: 0)) + guard let text else { return NSRange(location: 0, length: 0) } + let clampedOffset = min(max(0, offset), text.length) + return text.lineRangeBreakingAtLineEndings(for: NSRange(location: clampedOffset, length: 0)) } func lineAndColumn(forOffset offset: Int) -> (line: Int, column: Int) { - guard let textView else { return (0, 0) } - let nsString = textView.string as NSString - let clampedOffset = min(max(0, offset), nsString.length) - - if nsString.length == 0 { return (0, 0) } - - let safeOffset = min(clampedOffset, max(0, nsString.length - 1)) - let lineRange = nsString.lineRangeBreakingAtLineEndings( - for: NSRange(location: safeOffset, length: 0) - ) - let column = clampedOffset - lineRange.location - - // Count newlines before lineRange.location — uses fast NSString search - var line = 0 - var searchStart = 0 - while searchStart < lineRange.location { - let found = nsString.range(of: "\n", range: NSRange(location: searchStart, length: lineRange.location - searchStart)) - if found.location == NSNotFound { break } - line += 1 - searchStart = found.location + found.length - } - - return (line, column) + guard let text, text.length > 0 else { return (0, 0) } + let clampedOffset = min(max(0, offset), text.length) + // The end of the document is a column past the end of the last line, never the empty line after a final + // line ending. + let anchor = min(clampedOffset, text.length - 1) + let range = text.lineRangeBreakingAtLineEndings(for: NSRange(location: anchor, length: 0)) + return (lineIndex(of: range, in: text), clampedOffset - range.location) } func offset(forLine line: Int, column: Int) -> Int { - guard let textView else { return 0 } - let nsString = textView.string as NSString - var currentLine = 0 - var index = 0 - while index < nsString.length && currentLine < line { - let lineRange = nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) - currentLine += 1 - index = lineRange.location + lineRange.length - } - // Now index is at the start of the target line - let lineRange = nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: min(index, nsString.length), length: 0)) - // Content length excludes trailing newline + guard let text else { return 0 } + let range = lineRange(atIndex: max(0, line), in: text) + let lineEnd = NSMaxRange(range) let contentLength: Int - let lineEnd = lineRange.location + lineRange.length - if lineEnd > lineRange.location && lineEnd <= nsString.length - && nsString.character(at: lineEnd - 1) == 0x0A { - contentLength = lineRange.length - 1 + if lineEnd > range.location && lineEnd <= text.length && text.character(at: lineEnd - 1) == 0x0A { + contentLength = range.length - 1 } else { - contentLength = lineRange.length + contentLength = range.length } - let clampedCol = min(column, max(0, contentLength - 1)) - return lineRange.location + max(0, clampedCol) + let clampedColumn = min(column, max(0, contentLength - 1)) + return range.location + max(0, clampedColumn) } func character(at offset: Int) -> unichar { - guard let textView else { return 0 } - let nsString = textView.string as NSString - guard offset >= 0 && offset < nsString.length else { return 0 } - return nsString.character(at: offset) + guard let text, offset >= 0 && offset < text.length else { return 0 } + return text.character(at: offset) } func wordBoundary(forward: Bool, from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } + guard let text else { return offset } + guard text.length > 0 else { return 0 } if forward { - var pos = min(offset, nsString.length - 1) - let startClass = charClass(nsString.character(at: pos)) + var pos = min(offset, text.length - 1) + let startClass = charClass(text.character(at: pos)) if startClass == .whitespace { - // Skip whitespace, then stop at start of next word/punctuation - while pos < nsString.length && charClass(nsString.character(at: pos)) == .whitespace { + while pos < text.length && charClass(text.character(at: pos)) == .whitespace { pos += 1 } } else { - while pos < nsString.length && charClass(nsString.character(at: pos)) == startClass { + while pos < text.length && charClass(text.character(at: pos)) == startClass { pos += 1 } - while pos < nsString.length && charClass(nsString.character(at: pos)) == .whitespace { + while pos < text.length && charClass(text.character(at: pos)) == .whitespace { pos += 1 } } - return min(pos, nsString.length) + return min(pos, text.length) } else { - var pos = min(offset, nsString.length) + var pos = min(offset, text.length) if pos > 0 { pos -= 1 } - while pos > 0 && charClass(nsString.character(at: pos)) == .whitespace { + while pos > 0 && charClass(text.character(at: pos)) == .whitespace { pos -= 1 } - let cls = charClass(nsString.character(at: pos)) - while pos > 0 && charClass(nsString.character(at: pos - 1)) == cls { + let cls = charClass(text.character(at: pos)) + while pos > 0 && charClass(text.character(at: pos - 1)) == cls { pos -= 1 } return max(0, pos) @@ -153,20 +118,19 @@ final class VimTextBufferAdapter: VimTextBuffer { } func wordEnd(from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } + guard let text else { return offset } + guard text.length > 0 else { return 0 } - var pos = min(offset + 1, nsString.length - 1) - while pos < nsString.length && charClass(nsString.character(at: pos)) == .whitespace { + var pos = min(offset + 1, text.length - 1) + while pos < text.length && charClass(text.character(at: pos)) == .whitespace { pos += 1 } - guard pos < nsString.length else { return nsString.length - 1 } - let cls = charClass(nsString.character(at: pos)) - while pos < nsString.length - 1 && charClass(nsString.character(at: pos + 1)) == cls { + guard pos < text.length else { return text.length - 1 } + let cls = charClass(text.character(at: pos)) + while pos < text.length - 1 && charClass(text.character(at: pos + 1)) == cls { pos += 1 } - return min(pos, nsString.length - 1) + return min(pos, text.length - 1) } func selectedRange() -> NSRange { @@ -175,21 +139,20 @@ final class VimTextBufferAdapter: VimTextBuffer { } func string(in range: NSRange) -> String { - guard let textView else { return "" } - let nsString = textView.string as NSString + guard let text else { return "" } let clampedRange = NSRange( location: max(0, range.location), - length: min(range.length, nsString.length - max(0, range.location)) + length: min(range.length, text.length - max(0, range.location)) ) guard clampedRange.length > 0 else { return "" } - return nsString.substring(with: clampedRange) + return text.substring(with: clampedRange) } func setSelectedRange(_ range: NSRange) { guard let textView else { return } - let clampedLocation = max(0, min(range.location, (textView.string as NSString).length)) - let maxLength = (textView.string as NSString).length - clampedLocation - let clampedLength = max(0, min(range.length, maxLength)) + let documentLength = length + let clampedLocation = max(0, min(range.location, documentLength)) + let clampedLength = max(0, min(range.length, documentLength - clampedLocation)) let clampedRange = NSRange(location: clampedLocation, length: clampedLength) let currentRange = textView.selectedRange() @@ -215,104 +178,99 @@ final class VimTextBufferAdapter: VimTextBuffer { } func wordEndBackward(from offset: Int) -> Int { - guard let textView else { return 0 } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } - var pos = min(max(0, offset), nsString.length - 1) + guard let text else { return 0 } + guard text.length > 0 else { return 0 } + var pos = min(max(0, offset), text.length - 1) if pos > 0 { pos -= 1 } - if charClass(nsString.character(at: pos)) == .whitespace { - while pos > 0 && charClass(nsString.character(at: pos)) == .whitespace { + if charClass(text.character(at: pos)) == .whitespace { + while pos > 0 && charClass(text.character(at: pos)) == .whitespace { pos -= 1 } return pos } - let cls = charClass(nsString.character(at: pos)) - while pos > 0 && charClass(nsString.character(at: pos - 1)) == cls { + let cls = charClass(text.character(at: pos)) + while pos > 0 && charClass(text.character(at: pos - 1)) == cls { pos -= 1 } guard pos > 0 else { return 0 } pos -= 1 - while pos > 0 && charClass(nsString.character(at: pos)) == .whitespace { + while pos > 0 && charClass(text.character(at: pos)) == .whitespace { pos -= 1 } return pos } func bigWordBoundary(forward: Bool, from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } + guard let text else { return offset } + guard text.length > 0 else { return 0 } if forward { - var pos = min(offset, nsString.length - 1) - let startWS = isWhitespace(nsString.character(at: pos)) + var pos = min(offset, text.length - 1) + let startWS = isWhitespace(text.character(at: pos)) if startWS { - while pos < nsString.length && isWhitespace(nsString.character(at: pos)) { + while pos < text.length && isWhitespace(text.character(at: pos)) { pos += 1 } } else { - while pos < nsString.length && !isWhitespace(nsString.character(at: pos)) { + while pos < text.length && !isWhitespace(text.character(at: pos)) { pos += 1 } - while pos < nsString.length && isWhitespace(nsString.character(at: pos)) { + while pos < text.length && isWhitespace(text.character(at: pos)) { pos += 1 } } - return min(pos, nsString.length) + return min(pos, text.length) } - var pos = min(offset, nsString.length) + var pos = min(offset, text.length) if pos > 0 { pos -= 1 } - while pos > 0 && isWhitespace(nsString.character(at: pos)) { + while pos > 0 && isWhitespace(text.character(at: pos)) { pos -= 1 } - while pos > 0 && !isWhitespace(nsString.character(at: pos - 1)) { + while pos > 0 && !isWhitespace(text.character(at: pos - 1)) { pos -= 1 } return max(0, pos) } func bigWordEnd(from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } - var pos = min(offset + 1, nsString.length - 1) - while pos < nsString.length && isWhitespace(nsString.character(at: pos)) { + guard let text else { return offset } + guard text.length > 0 else { return 0 } + var pos = min(offset + 1, text.length - 1) + while pos < text.length && isWhitespace(text.character(at: pos)) { pos += 1 } - guard pos < nsString.length else { return nsString.length - 1 } - while pos < nsString.length - 1 && !isWhitespace(nsString.character(at: pos + 1)) { + guard pos < text.length else { return text.length - 1 } + while pos < text.length - 1 && !isWhitespace(text.character(at: pos + 1)) { pos += 1 } - return min(pos, nsString.length - 1) + return min(pos, text.length - 1) } func bigWordEndBackward(from offset: Int) -> Int { - guard let textView else { return 0 } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } - var pos = min(max(0, offset), nsString.length - 1) + guard let text else { return 0 } + guard text.length > 0 else { return 0 } + var pos = min(max(0, offset), text.length - 1) if pos > 0 { pos -= 1 } - if isWhitespace(nsString.character(at: pos)) { - while pos > 0 && isWhitespace(nsString.character(at: pos)) { + if isWhitespace(text.character(at: pos)) { + while pos > 0 && isWhitespace(text.character(at: pos)) { pos -= 1 } return pos } - while pos > 0 && !isWhitespace(nsString.character(at: pos - 1)) { + while pos > 0 && !isWhitespace(text.character(at: pos - 1)) { pos -= 1 } guard pos > 0 else { return 0 } pos -= 1 - while pos > 0 && isWhitespace(nsString.character(at: pos)) { + while pos > 0 && isWhitespace(text.character(at: pos)) { pos -= 1 } return pos } func matchingBracket(at offset: Int) -> Int? { - guard let textView else { return nil } - let nsString = textView.string as NSString - guard offset >= 0 && offset < nsString.length else { return nil } - let ch = nsString.character(at: offset) + guard let text else { return nil } + guard offset >= 0 && offset < text.length else { return nil } + let ch = text.character(at: offset) let pairs: [unichar: (close: unichar, forward: Bool)] = [ 0x28: (0x29, true), 0x5B: (0x5D, true), 0x7B: (0x7D, true), 0x29: (0x28, false), 0x5D: (0x5B, false), 0x7D: (0x7B, false) @@ -321,8 +279,8 @@ final class VimTextBufferAdapter: VimTextBuffer { let step = pair.forward ? 1 : -1 var depth = 1 var pos = offset + step - while pos >= 0 && pos < nsString.length { - let cur = nsString.character(at: pos) + while pos >= 0 && pos < text.length { + let cur = text.character(at: pos) if cur == ch { depth += 1 } else if cur == pair.close { @@ -339,12 +297,12 @@ final class VimTextBufferAdapter: VimTextBuffer { return (0, max(0, lineCount - 1)) } let visible = scrollView.documentVisibleRect - let nsString = textView.string as NSString - guard nsString.length > 0 else { return (0, 0) } + let documentLength = length + guard documentLength > 0 else { return (0, 0) } let topGlyph = textView.layoutManager.textOffsetAtPoint(visible.origin) ?? 0 let bottomGlyph = textView.layoutManager.textOffsetAtPoint( CGPoint(x: visible.origin.x, y: visible.maxY - 1) - ) ?? max(0, nsString.length - 1) + ) ?? max(0, documentLength - 1) let topLine = lineAndColumn(forOffset: topGlyph).line let bottomLine = lineAndColumn(forOffset: bottomGlyph).line return (min(topLine, bottomLine), max(topLine, bottomLine)) @@ -358,6 +316,53 @@ final class VimTextBufferAdapter: VimTextBuffer { AppSettingsManager.shared.editor.clampedTabWidth } + // MARK: - Line Storage + + // The layout manager's line storage finds a line by index or offset in O(log n) and breaks lines where + // `lineRangeBreakingAtLineEndings` does. While an edit is still being applied it describes the old text, and an + // edit across a CR LF pair can leave it off the text's lines, so either case sends the query back to the text. + private func lineStorage(matching text: NSString) -> TextLineStorage? { + guard let storage = textView?.textStorage, let lines = textView?.layoutManager?.lineStorage, !lines.isEmpty, + lines.length == text.length, !storage.editedMask.contains(.editedCharacters) else { + return nil + } + if let trust = lineStorageTrust, trust.storage === storage, trust.length == text.length { + return trust.trusted ? lines : nil + } + let trusted = text.range(of: "\r", options: .literal).location == NSNotFound || lines.breaksLines(like: text) + lineStorageTrust = LineStorageTrust(storage: storage, length: text.length, trusted: trusted) + return trusted ? lines : nil + } + + private func lineIndex(of range: NSRange, in text: NSString) -> Int { + if let line = lineStorage(matching: text)?.getLine(atOffset: range.location), line.range == range { + return line.index + } + var index = 0 + var location = 0 + while location < range.location { + location = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0))) + index += 1 + } + return index + } + + // An index past the last line resolves to the last line, which is the empty one after a final line ending. + private func lineRange(atIndex index: Int, in text: NSString) -> NSRange { + if let lines = lineStorage(matching: text), + let line = lines.getLine(atIndex: min(index, lines.count - 1)), + line.range == text.lineRangeBreakingAtLineEndings(for: NSRange(location: line.range.location, length: 0)) { + return line.range + } + var current = 0 + var location = 0 + while location < text.length && current < index { + location = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0))) + current += 1 + } + return text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0)) + } + // MARK: - Helpers private enum CharClass { @@ -376,4 +381,8 @@ final class VimTextBufferAdapter: VimTextBuffer { private func isWhitespace(_ char: unichar) -> Bool { char == 0x20 || char == 0x09 || char == 0x0A || char == 0x0D } + + private func isLineBreak(_ char: unichar) -> Bool { + char == 0x0A || char == 0x0D + } } diff --git a/apple/TablePro/Extensions/TextLineStorage+LineBreaks.swift b/apple/TablePro/Extensions/TextLineStorage+LineBreaks.swift new file mode 100644 index 0000000000..e1571c672d --- /dev/null +++ b/apple/TablePro/Extensions/TextLineStorage+LineBreaks.swift @@ -0,0 +1,28 @@ +import Foundation +import TableProTextEngine + +extension TextLineStorage where Data == TextLine { + // The layout manager updates its lines edit by edit and can split or join a CR LF pair an edit crosses, after + // which its lines no longer end where the text's line breaks do. Without a CR in the text that cannot happen. + func breaksLines(like text: NSString) -> Bool { + guard length == text.length else { return false } + let lineBreaks = CharacterSet(charactersIn: "\r\n") + var start = 0 + var index = 0 + while let line = getLine(atIndex: index), line.range.location == start { + let rest = NSRange(location: start, length: text.length - start) + let found = text.rangeOfCharacter(from: lineBreaks, options: .literal, range: rest) + guard found.location != NSNotFound else { + return NSMaxRange(line.range) == text.length && index == count - 1 + } + var end = NSMaxRange(found) + if text.character(at: found.location) == 0x0D, end < text.length, text.character(at: end) == 0x0A { + end += 1 + } + guard NSMaxRange(line.range) == end else { return false } + start = end + index += 1 + } + return false + } +} diff --git a/apple/TablePro/Models/Query/QueryHistoryEntry.swift b/apple/TablePro/Models/Query/QueryHistoryEntry.swift index 139776f907..5355cb10f0 100644 --- a/apple/TablePro/Models/Query/QueryHistoryEntry.swift +++ b/apple/TablePro/Models/Query/QueryHistoryEntry.swift @@ -97,26 +97,95 @@ struct QueryHistoryEntry: Identifiable, Codable, Hashable, Sendable { ) } + // Reads only the ends of the query: the Quick Switcher builds this for every history item it lists. var queryPreview: String { - var trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmable = CharacterSet.whitespacesAndNewlines + let scalars = query.unicodeScalars + guard let first = scalars.firstIndex(where: { !trimmable.contains($0) }), + let last = scalars.lastIndex(where: { !trimmable.contains($0) }) else { + return ";" + } + var head = PreviewHead(limit: 100) + for scalar in scalars[first...last] { + head.append(scalar) + if head.isComplete { return head.preview } + } + var trimmed = head.text if !trimmed.hasSuffix(";") { trimmed += ";" } - if (trimmed as NSString).length > 100 { - return String(trimmed.prefix(100)) + "…" - } - return trimmed + return PreviewHead.shortened(trimmed, to: head.limit) } + // Stops reading once the cut is known: a history row builds this every time it renders. var singleLinePreview: String { - let collapsed = query - .components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty } - .joined(separator: " ") - if (collapsed as NSString).length > 160 { - return String(collapsed.prefix(160)) + "…" + let newlines = CharacterSet.newlines + let whitespaces = CharacterSet.whitespaces + let scalars = query.unicodeScalars + var head = PreviewHead(limit: 160) + var lineHasText = false + var gapStart: String.UnicodeScalarView.Index? + var index = scalars.startIndex + while index < scalars.endIndex { + let scalar = scalars[index] + if newlines.contains(scalar) { + lineHasText = false + gapStart = nil + } else if whitespaces.contains(scalar) { + if lineHasText && gapStart == nil { + gapStart = index + } + } else { + if !lineHasText { + lineHasText = true + if !head.text.isEmpty { + head.append(" ") + } + } else if let start = gapStart { + gapStart = nil + for gapScalar in scalars[start..= nextCount else { return } + nextCount = scalarCount * 2 + isComplete = text.count > limit + } + + // Tests the UTF-16 length but cuts at a Character count, as the previews always have. + static func shortened(_ text: String, to limit: Int) -> String { + guard (text as NSString).length > limit else { return text } + return String(text.prefix(limit)) + "…" } } diff --git a/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift b/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift index c4d0956510..96f13af9d1 100644 --- a/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift +++ b/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift @@ -1,18 +1,11 @@ -// -// SQLLineFoldProvider.swift -// TablePro -// - import Foundation import TableProEditorKit import TableProSQLGrammar import TableProTextEngine import TableProUtilities -/// Answers the editor's per-line fold queries from a whole-document scan. -/// -/// The editor restarts its walk at line zero every time the document changes, so the scan is refreshed there and every -/// other line is a dictionary lookup. That keeps the main-thread cost of a pass linear in the document, not quadratic. +// The editor runs `documentFoldScanner()` off the main thread on a copy of the text. The per-line answers below are +// for callers that ask a line at a time; they rescan at line zero, so a pass stays linear in the document. final class SQLLineFoldProvider: LineFoldProvider { var grammar: SQLLexicalGrammar @@ -23,6 +16,15 @@ final class SQLLineFoldProvider: LineFoldProvider { self.grammar = grammar } + func documentFoldScanner() -> LineFoldDocumentScanner? { + let grammar = grammar + return { text in + SQLFoldScanner.scan(text as NSString, grammar: grammar).regions.map { + LineFoldProviderFold(depth: $0.depth, range: $0.range) + } + } + } + func foldLevelAtLine( lineNumber: Int, lineRange: NSRange, diff --git a/apple/TablePro/Views/Editor/SQLEditorController.swift b/apple/TablePro/Views/Editor/SQLEditorController.swift index e62f529488..0c3cf2a8df 100644 --- a/apple/TablePro/Views/Editor/SQLEditorController.swift +++ b/apple/TablePro/Views/Editor/SQLEditorController.swift @@ -164,6 +164,7 @@ final class SQLEditorController { coordinator.onAIAction = input.onAIAction coordinator.onSaveAsFavorite = input.onSaveAsFavorite coordinator.schemaProvider = input.schemaProvider + coordinator.databaseScope = input.databaseScope coordinator.databaseType = input.databaseType coordinator.tabID = input.tabID coordinator.connectionId = input.connectionId diff --git a/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift b/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift index 1ec3d48dc9..f48e6c8f84 100644 --- a/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift +++ b/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift @@ -35,6 +35,7 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie private let statementRunController = StatementRunController() /// Shared schema provider for inline AI suggestions (avoids duplicate schema fetches) var schemaProvider: SQLSchemaProvider? + var databaseScope: DatabaseScope? private let inlineAccessGate: AIConnectionAccessGate = .live private var contextMenu: AIEditorContextMenu? private var inlineSuggestionManager: InlineSuggestionManager? @@ -245,12 +246,6 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie self?.vimCursorManager?.updatePosition() } - if !isLargeDocument, !didDestroy, let tabID, let sync = copilotDocumentSync, - resolvedInlineSourceKind == .copilot { - let text = textView.string - Task { await sync.didChangeText(tabID: tabID, newText: text) } - } - frameChangeTask?.cancel() frameChangeTask = Task { [weak controller] in try? await Task.sleep(for: .milliseconds(50)) @@ -267,6 +262,7 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie vimEngine?.invalidateLineCache() foldPreview.dismiss() inlineSuggestionManager?.dismissSuggestion() + copilotDocumentSync?.textViewDidReplaceDocument() statementRunController.refreshControls(in: controller) statementRunController.refreshHighlight(in: controller) diagnosticsController.clear(in: controller) @@ -320,10 +316,7 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie removeVimKeyInterceptor() removeEmacsKeyInterceptor() - if let tabID, let sync = copilotDocumentSync { - let id = tabID - Task { await sync.didCloseTab(tabID: id) } - } + copilotDocumentSync?.detach() foldPreview.destroy() inlineSuggestionManager?.uninstall() @@ -620,44 +613,61 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie } private func installCopilotInlineSource() { - let sync = CopilotDocumentSync() + let sync = CopilotDocumentSync(server: LiveCopilotDocumentServer(), lengthLimit: Self.languageServiceLengthLimit) copilotDocumentSync = sync copilotInlineSource = CopilotInlineSource(documentSync: sync) - let capturedTabID = tabID - let capturedText = controller?.textView?.string ?? "" - let capturedSchemaProvider = schemaProvider - let capturedDBType = databaseType - let dbName = connectionId.flatMap { - DatabaseManager.shared.session(for: $0)?.resolvedBrowseDatabase - } ?? "database" + let referenceText: String? + if let textView = controller?.textView, textView.textStorage.length <= Self.languageServiceLengthLimit { + referenceText = textView.string + } else { + referenceText = nil + } + let source = copilotSchemaSource() Task { [weak self] in - if let provider = capturedSchemaProvider, let dbType = capturedDBType { + if let source { await sync.preambleBuilder.buildPreamble( - schemaProvider: provider, - databaseName: dbName, - databaseType: dbType + schemaProvider: source.schemaProvider, + databaseName: source.databaseName, + databaseType: source.databaseType, + referencing: referenceText, + grammar: source.grammar, + maxTables: source.budget ) } guard let self, self.copilotDocumentSync === sync, self.resolvedInlineSourceKind == .copilot else { return } await CopilotService.shared.start() - guard self.copilotDocumentSync === sync else { return } - if let tabID = capturedTabID { - sync.ensureDocumentOpen(tabID: tabID, text: capturedText) - await sync.didActivateTab(tabID: tabID, text: capturedText) - } + guard self.copilotDocumentSync === sync, let textView = self.controller?.textView else { return } + sync.attach( + to: textView, + tabID: { [weak self] in self?.tabID }, + isAllowed: { [weak self] in self?.resolvedInlineSourceKind == .copilot }, + schemaSource: { [weak self] in self?.copilotSchemaSource() } + ) } } + private func copilotSchemaSource() -> CopilotPreambleBuilder.Source? { + guard let schemaProvider, let databaseType else { return nil } + let scopeDatabase = databaseScope.flatMap { $0.isServerScoped ? nil : $0.database } + let databaseName = scopeDatabase ?? connectionId.flatMap { + DatabaseManager.shared.session(for: $0)?.resolvedBrowseDatabase + } ?? "database" + return CopilotPreambleBuilder.Source( + schemaProvider: schemaProvider, + databaseName: databaseName, + databaseType: databaseType, + grammar: lexicalGrammar, + budget: max(AppSettingsManager.shared.ai.maxSchemaTables, 0) + ) + } + private func teardownInlineSources(except kind: InlineSuggestionSourceKind) { if kind != .copilot { - if let tabID, let sync = copilotDocumentSync { - let id = tabID - Task { await sync.didCloseTab(tabID: id) } - } + copilotDocumentSync?.detach() copilotDocumentSync = nil copilotInlineSource = nil } diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index d603b2f471..281a1bb143 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -266,7 +266,9 @@ final class MainContentCoordinator: ObservableObject { // MARK: - Published State - @Published var cursorPositions: [CursorPosition] = [] + // Not published: the editor writes it on every caret move, and a publish re-rendered every view observing this + // coordinator. Readers take it when they need it. + var cursorPositions: [CursorPosition] = [] @Published var activeSheet: ActiveSheet? { didSet { releaseImportFileUnlessImporting() } } diff --git a/apple/TableProTests/Core/AI/CopilotDocumentSyncTests.swift b/apple/TableProTests/Core/AI/CopilotDocumentSyncTests.swift new file mode 100644 index 0000000000..19e060ebf0 --- /dev/null +++ b/apple/TableProTests/Core/AI/CopilotDocumentSyncTests.swift @@ -0,0 +1,555 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProEditorKit +import TableProTestSupport +import TableProTextEngine +import Testing +import TextStory + +@MainActor +private final class RecordingCopilotServer: CopilotDocumentServer { + enum Message { + case open(LSPTextDocumentItem) + case change(LSPDidChangeParams) + case focus(String) + case close(String) + } + + var generation = 1 + var hasClient = true + private(set) var messages: [Message] = [] + + var changes: [LSPDidChangeParams] { + messages.compactMap { message in + guard case .change(let params) = message else { return nil } + return params + } + } + + func didOpen(_ document: LSPTextDocumentItem) async { + messages.append(.open(document)) + } + + func didChange(_ change: LSPDidChangeParams) async { + messages.append(.change(change)) + } + + func didFocus(uri: String) async { + messages.append(.focus(uri)) + } + + func didClose(uri: String) async { + messages.append(.close(uri)) + } +} + +@MainActor +private final class TabSelection { + var id = UUID() +} + +@MainActor +private final class SchemaSelection { + var provider: SQLSchemaProvider + var budget: Int + + init(provider: SQLSchemaProvider, budget: Int) { + self.provider = provider + self.budget = budget + } + + var source: CopilotPreambleBuilder.Source { + CopilotPreambleBuilder.Source( + schemaProvider: provider, + databaseName: "shop", + databaseType: .mysql, + grammar: DatabaseType.mysql.lexicalGrammar, + budget: budget + ) + } +} + +private final class FetchCounter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func increment() { + lock.withLock { value += 1 } + } +} + +// The server's document model (vscode-languageserver-textdocument): CR, LF and CR LF end a line, and a position past +// a line's content is clamped to before its line break. +private struct LSPDocumentModel { + private var units: [UInt16] + + init(_ text: String) { + units = Array(text.utf16) + } + + var text: String { + String(decoding: units, as: UTF16.self) + } + + mutating func apply(_ change: LSPTextDocumentContentChangeEvent) { + let replacement = Array(change.text.utf16) + guard let range = change.range else { + units = replacement + return + } + let start = offset(at: range.start) + let end = max(start, offset(at: range.end)) + units.replaceSubrange(start.. Int { + let lineStarts = lineStarts() + guard position.line < lineStarts.count else { return units.count } + guard position.line >= 0 else { return 0 } + let lineStart = lineStarts[position.line] + guard position.character > 0 else { return lineStart } + let nextLineStart = position.line + 1 < lineStarts.count ? lineStarts[position.line + 1] : units.count + var offset = min(lineStart + position.character, nextLineStart) + while offset > lineStart, units[offset - 1] == 0x0D || units[offset - 1] == 0x0A { + offset -= 1 + } + return offset + } + + private func lineStarts() -> [Int] { + var starts = [0] + var index = 0 + while index < units.count { + let unit = units[index] + if unit == 0x0D || unit == 0x0A { + if unit == 0x0D, index + 1 < units.count, units[index + 1] == 0x0A { + index += 1 + } + starts.append(index + 1) + } + index += 1 + } + return starts + } +} + +@MainActor +struct CopilotDocumentSyncTests { + @MainActor + private struct Harness { + let controller: TextViewController + let server: RecordingCopilotServer + let sync: CopilotDocumentSync + let tab: TabSelection + let schema: SchemaSelection + + var textView: TextView { controller.textView } + var uri: String { sync.documentURI(for: tab.id) } + var expectedText: String { sync.preambleBuilder.preamble + controller.textView.string } + + func replayed(uri: String? = nil) -> String? { + CopilotDocumentSyncTests.replayedText(of: server.messages, uri: uri ?? self.uri) + } + } + + @Test("random edits, typing filter edits and a multi-range replace replay to the editor's text as range changes") + func incrementalEditsReplayToTheEditorText() async throws { + let harness = await Self.makeHarness(text: "SELECT id, name\nFROM table_0\nWHERE id = 1;\n") + var generator = SeededGenerator(seed: 0x5EED) + let snippets = ["x", "SELECT ", "\n", "a\nb", "é", "😀", " ", "\n\n", "(", "'", ";\n"] + + for step in 0..<120 { + let text = harness.textView.string as NSString + let location = Self.boundary(generator.index(below: text.length + 1), in: text) + let end = Self.boundary(min(text.length, location + 1 + generator.index(below: 8)), in: text) + let snippet = snippets[generator.index(below: snippets.count)] + switch generator.index(below: 3) { + case 0: + harness.textView.replaceCharacters(in: NSRange(location: location, length: 0), with: snippet) + case 1: + harness.textView.replaceCharacters(in: NSRange(location: location, length: end - location), with: "") + default: + harness.textView.replaceCharacters(in: NSRange(location: location, length: end - location), with: snippet) + } + if step.isMultiple(of: 10) { + await harness.sync.flush() + } + } + + let text = harness.textView.string as NSString + harness.textView.replaceCharacters([ + TextReplacement(range: NSRange(location: 0, length: 0), string: "-- head\n"), + TextReplacement(range: NSRange(location: Self.boundary(text.length / 2, in: text), length: 0), string: "mid\n"), + TextReplacement(range: NSRange(location: text.length, length: 0), string: "\n-- tail") + ]) + harness.textView.applyMutation( + TextMutation(string: "/* filter */\n", range: NSRange(location: 0, length: 0), limit: harness.textView.textStorage.length) + ) + await harness.sync.flush() + + let sent = harness.server.changes + try #require(!sent.isEmpty) + #expect(sent.allSatisfy { params in params.contentChanges.allSatisfy { $0.range != nil } }) + let versions = sent.map { $0.textDocument.version } + #expect(versions == Array(1...sent.count)) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a keystroke in a 130,000 unit script with a 300-table preamble sends under 1 KB") + func keystrokePayloadStaysSmall() async throws { + let document = Self.script(units: 130_000) + let harness = await Self.makeHarness(text: document, tableCount: 300) + let fullText = LSPDidChangeParams( + textDocument: LSPVersionedTextDocumentIdentifier(uri: harness.uri, version: 1), + contentChanges: [LSPTextDocumentContentChangeEvent(text: harness.sync.preambleBuilder.prependToText(document))] + ) + let fullTextBytes = try JSONEncoder().encode(fullText).count + + harness.textView.replaceCharacters(in: NSRange(location: 65_000, length: 0), with: "x") + await harness.sync.flush() + + let keystroke = try #require(harness.server.changes.last) + let keystrokeBytes = try JSONEncoder().encode(keystroke).count + #expect(keystroke.contentChanges.first?.range != nil) + #expect(keystrokeBytes < 1_024) + #expect(fullTextBytes > 200_000) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("typing in a CR LF script stays incremental; an edit inside a CR LF sends the full text until the lines agree again") + func editNextToCarriageReturnSendsFullText() async throws { + let harness = await Self.makeHarness(text: "SELECT id\r\nFROM table_0\r\nWHERE id = 1\r\n") + + harness.textView.replaceCharacters(in: NSRange(location: 13, length: 0), with: "x") + await harness.sync.flush() + let typed = try #require(harness.server.changes.last) + #expect(typed.contentChanges.allSatisfy { $0.range != nil }) + #expect(harness.replayed() == harness.expectedText) + + let lineFeed = (harness.textView.string as NSString).range(of: "\n", options: .literal).location + harness.textView.replaceCharacters(in: NSRange(location: lineFeed, length: 0), with: "y") + await harness.sync.flush() + let split = try #require(harness.server.changes.last) + #expect(split.contentChanges.count == 1) + #expect(split.contentChanges.first?.range == nil) + #expect(harness.replayed() == harness.expectedText) + + harness.textView.replaceCharacters(in: NSRange(location: harness.textView.textStorage.length, length: 0), with: "LIMIT 1") + await harness.sync.flush() + let later = try #require(harness.server.changes.last) + #expect(later.contentChanges.first?.range == nil) + #expect(harness.replayed() == harness.expectedText) + + harness.textView.replaceCharacters(in: NSRange(location: lineFeed, length: 1), with: "") + await harness.sync.flush() + let rejoined = try #require(harness.server.changes.last) + #expect(rejoined.contentChanges.first?.range == nil) + + harness.textView.replaceCharacters(in: NSRange(location: harness.textView.textStorage.length, length: 0), with: ";") + await harness.sync.flush() + let resumed = try #require(harness.server.changes.last) + #expect(resumed.contentChanges.allSatisfy { $0.range != nil }) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("switching tabs sends nothing; the first edit or request there closes the old document and opens the new one") + func tabSwitchSendsNothingUntilTheTabIsUsed() async { + let harness = await Self.makeHarness(text: "SELECT 1;\n") + let firstURI = harness.uri + let before = harness.server.messages.count + + harness.tab.id = UUID() + harness.controller.setText("SELECT name FROM table_1;\n") + harness.sync.textViewDidReplaceDocument() + #expect(harness.sync.currentDocumentInfo()?.uri == nil) + + harness.textView.replaceCharacters(in: NSRange(location: 7, length: 0), with: "id, ") + await harness.sync.flush() + let secondURI = harness.uri + #expect(secondURI != firstURI) + #expect(Self.summary(Array(harness.server.messages.dropFirst(before))) == [ + "close \(firstURI)", "open \(secondURI)", "focus \(secondURI)" + ]) + #expect(harness.sync.currentDocumentInfo()?.uri == secondURI) + #expect(harness.replayed(uri: secondURI) == harness.expectedText) + #expect(harness.replayed(uri: firstURI) == nil) + + let switched = harness.server.messages.count + harness.tab.id = UUID() + harness.controller.setText("SELECT 3;\n") + harness.sync.textViewDidReplaceDocument() + await harness.sync.flush() + #expect(Self.summary(Array(harness.server.messages.dropFirst(switched))) == [ + "close \(secondURI)", "open \(harness.uri)", "focus \(harness.uri)" + ]) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a request whose statement names an undescribed table puts it in the preamble and resends the full text once") + func requestAddsTheStatementsTableToThePreamble() async throws { + let fetches = FetchCounter() + let harness = await Self.makeHarness(text: "SELECT 1;\n", tableCount: 2_000, maxTables: 20, fetches: fetches) + #expect(!harness.sync.preambleBuilder.preamble.contains("-- table_1500(")) + harness.textView.replaceCharacters(in: NSRange(location: 0, length: 9), with: "SELECT * FROM table_1500;") + await harness.sync.flush() + let before = harness.server.messages.count + let fetchesBefore = fetches.count + + harness.sync.refreshPreamble(forStatementAt: 10, in: harness.textView.string) + try await waitUntil { harness.sync.preambleBuilder.preamble.contains("-- table_1500(") } + await harness.sync.flush() + + #expect(fetches.count - fetchesBefore <= 20) + let tail = Array(harness.server.messages.dropFirst(before)) + #expect(Self.summary(tail) == ["change \(harness.uri)"]) + let resent = try #require(harness.server.changes.last) + #expect(resent.contentChanges.count == 1) + #expect(resent.contentChanges.first?.range == nil) + #expect(resent.contentChanges.first?.text.contains("-- table_1500(") == true) + #expect(harness.replayed() == harness.expectedText) + + let settled = harness.server.messages.count + let described = await harness.sync.preambleBuilder.rebuiltPreamble( + forStatementAt: 10, + in: harness.textView.string, + from: harness.schema.source + ) + #expect(described?.preamble == nil) + await harness.sync.flush() + #expect(harness.server.messages.count == settled) + } + + @Test("an editor that had no schema when Copilot started gets the schema header on its first request") + func requestBuildsThePreambleAnEditorStartedWithout() async throws { + let harness = await Self.makeHarness(text: "SELECT * FROM table_1;\n", buildsPreamble: false) + #expect(harness.sync.preambleBuilder.preamble.isEmpty) + let before = harness.server.messages.count + + harness.sync.refreshPreamble(forStatementAt: 10, in: harness.textView.string) + try await waitUntil { harness.sync.preambleBuilder.preamble.contains("-- table_1(") } + await harness.sync.flush() + + #expect(harness.server.messages.count == before + 1) + let resent = try #require(harness.server.changes.last) + #expect(resent.contentChanges.first?.range == nil) + #expect(resent.contentChanges.first?.text.hasPrefix("-- Database: shop\n") == true) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("after the tab moves to another database, a request describes that database's table of the same name") + func requestFollowsTheTabsSchemaProvider() async throws { + let sales = await Self.makeProvider( + tables: [TestFixtures.makeTableInfo(name: "orders")], + columns: [TestFixtures.makeColumnInfo(name: "amount", dataType: "DECIMAL")] + ) + let staff = await Self.makeProvider( + tables: [TestFixtures.makeTableInfo(name: "orders")], + columns: [TestFixtures.makeColumnInfo(name: "salary", dataType: "INT")] + ) + let harness = await Self.makeHarness(text: "SELECT * FROM orders;\n", provider: sales) + #expect(harness.sync.preambleBuilder.preamble.contains("-- orders(amount DECIMAL PK NOT NULL)")) + let before = harness.server.messages.count + + harness.schema.provider = staff + harness.sync.refreshPreamble(forStatementAt: 10, in: harness.textView.string) + try await waitUntil { harness.sync.preambleBuilder.preamble.contains("salary") } + await harness.sync.flush() + + let resent = try #require(harness.server.changes.last) + #expect(harness.server.messages.count == before + 1) + #expect(resent.contentChanges.first?.range == nil) + let text = try #require(resent.contentChanges.first?.text) + #expect(text.contains("-- orders(salary INT PK NOT NULL)")) + #expect(!text.contains("amount")) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a lower Max schema tables applies on the next request") + func lowerBudgetAppliesOnTheNextRequest() async throws { + let harness = await Self.makeHarness(text: "SELECT 1;\n", tableCount: 10) + #expect(harness.sync.preambleBuilder.preamble.contains("-- table_9(")) + + harness.schema.budget = 3 + harness.sync.refreshPreamble(forStatementAt: 3, in: harness.textView.string) + try await waitUntil { !harness.sync.preambleBuilder.preamble.contains("-- table_9(") } + await harness.sync.flush() + + let text = try #require(harness.server.changes.last?.contentChanges.first?.text) + #expect(text.contains("-- … and 7 more tables")) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a tab switch that keeps the same text moves the next edit to the new tab's document") + func tabSwitchWithTheSameTextMovesTheNextEdit() async { + let harness = await Self.makeHarness(text: "SELECT 1;\n") + let firstURI = harness.uri + let before = harness.server.messages.count + + harness.tab.id = UUID() + harness.textView.replaceCharacters(in: NSRange(location: 8, length: 0), with: ", 2") + await harness.sync.flush() + + #expect(Self.summary(Array(harness.server.messages.dropFirst(before))) == [ + "close \(firstURI)", "open \(harness.uri)", "focus \(harness.uri)" + ]) + #expect(harness.sync.currentDocumentInfo()?.uri == harness.uri) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("after the server restarts, the next edit or request reopens the document with its full text") + func serverRestartReopensTheDocument() async { + let harness = await Self.makeHarness(text: "SELECT 1;\n") + let before = harness.server.messages.count + + harness.server.hasClient = false + harness.server.generation += 1 + harness.textView.replaceCharacters(in: NSRange(location: 8, length: 0), with: ", 2") + await harness.sync.flush() + #expect(harness.server.messages.count == before) + #expect(harness.sync.currentDocumentInfo()?.uri == nil) + + harness.server.hasClient = true + harness.server.generation += 1 + harness.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "-- again\n") + await harness.sync.flush() + #expect(Self.summary(Array(harness.server.messages.dropFirst(before))) == ["open \(harness.uri)", "focus \(harness.uri)"]) + #expect(harness.sync.currentDocumentInfo()?.version == 0) + #expect(harness.replayed() == harness.expectedText) + + let reopened = harness.server.messages.count + harness.server.generation += 1 + await harness.sync.flush() + #expect(Self.summary(Array(harness.server.messages.dropFirst(reopened))) == ["open \(harness.uri)", "focus \(harness.uri)"]) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("past the length limit nothing is sent, and back under it the full text is") + func overTheLengthLimitResyncsOnceUnder() async throws { + let harness = await Self.makeHarness(text: "SELECT 1;", lengthLimit: 20) + let before = harness.server.messages.count + + harness.textView.replaceCharacters(in: NSRange(location: 9, length: 0), with: " -- a long comment") + await harness.sync.flush() + #expect(harness.server.messages.count == before) + #expect(harness.sync.currentDocumentInfo()?.uri == nil) + + harness.textView.replaceCharacters(in: NSRange(location: 9, length: 18), with: "") + await harness.sync.flush() + let resync = try #require(harness.server.changes.last) + #expect(resync.contentChanges.count == 1) + #expect(resync.contentChanges.first?.range == nil) + #expect(harness.replayed() == harness.expectedText) + } + + // MARK: - Helpers + + private static func makeHarness( + text: String, + tableCount: Int = 3, + maxTables: Int? = nil, + lengthLimit: Int = 2_000_000, + fetches: FetchCounter? = nil, + provider given: SQLSchemaProvider? = nil, + buildsPreamble: Bool = true + ) async -> Harness { + let server = RecordingCopilotServer() + let sync = CopilotDocumentSync(server: server, lengthLimit: lengthLimit) + let provider: SQLSchemaProvider + if let given { + provider = given + } else { + provider = await makeProvider( + tables: (0.. SQLSchemaProvider { + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { _, _ in + fetches?.increment() + return columns + }, + fetchAllColumns: { [:] } + ) + let provider = SQLSchemaProvider(metadataSource: source) + await provider.resetForDatabase( + "shop", + tables: tables, + driver: MockDatabaseDriver(), + connection: TestFixtures.makeConnection() + ) + await provider.waitForEagerColumnLoad() + return provider + } + + private static func script(units: Int) -> String { + let block = "CREATE TABLE t (\n id INT,\n name TEXT\n);\nSELECT id, name\n FROM t\n WHERE id IN (\n 1, 2, 3\n );\n" + return String(repeating: block, count: units / (block as NSString).length + 1) + } + + private static func boundary(_ offset: Int, in text: NSString) -> Int { + guard offset > 0, offset < text.length else { return offset } + return text.rangeOfComposedCharacterSequence(at: offset).location + } + + private static func summary(_ messages: [RecordingCopilotServer.Message]) -> [String] { + messages.map { message -> String in + switch message { + case .open(let document): "open \(document.uri)" + case .change(let params): "change \(params.textDocument.uri)" + case .focus(let uri): "focus \(uri)" + case .close(let uri): "close \(uri)" + } + } + } + + private static func replayedText(of messages: [RecordingCopilotServer.Message], uri: String) -> String? { + var model: LSPDocumentModel? + for message in messages { + switch message { + case .open(let document) where document.uri == uri: + model = LSPDocumentModel(document.text) + case .change(let params) where params.textDocument.uri == uri: + for change in params.contentChanges { + model?.apply(change) + } + case .close(let closed) where closed == uri: + model = nil + default: + break + } + } + return model?.text + } +} diff --git a/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift b/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift index 027662991b..87935e50d9 100644 --- a/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift +++ b/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift @@ -8,10 +8,21 @@ import Foundation import TableProPluginKit import Testing +private final class FetchCounter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func increment() { + lock.withLock { value += 1 } + } +} + @MainActor struct CopilotPreambleBuilderTests { - /// The preamble fetched every table's columns with no schema and keyed them by the bare name, - /// so a table outside the current schema listed nothing or its namesake's columns. @Test("same-named tables in two schemas each list their own columns under a qualified name") func sameNamedTablesInTwoSchemasListTheirOwnColumns() async { let columnsBySchema: [String: [ColumnInfo]] = [ @@ -35,10 +46,52 @@ struct CopilotPreambleBuilderTests { await provider.waitForEagerColumnLoad() let builder = CopilotPreambleBuilder() - await builder.buildPreamble(schemaProvider: provider, databaseName: "shop", databaseType: .mysql) + await builder.buildPreamble(schemaProvider: provider, databaseName: "shop", databaseType: .mysql, maxTables: 20) let lines = builder.preamble.split(separator: "\n").map(String.init) #expect(lines.contains("-- sales.orders(amount DECIMAL PK NOT NULL)")) #expect(lines.contains("-- hr.orders(salary INT PK NOT NULL)")) } + + @Test("a large catalog reads columns for the table budget only, the document's tables first, one line per table") + func largeCatalogReadsColumnsForTheBudgetOnly() async throws { + let fetches = FetchCounter() + let columns = [TestFixtures.makeColumnInfo(name: "id")] + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { _, _ in + fetches.increment() + return columns + }, + fetchAllColumns: { [:] } + ) + let provider = SQLSchemaProvider(metadataSource: source) + let tables = [TestFixtures.makeTableInfo(name: "line\nbreak")] + + (1..<2_000).map { TestFixtures.makeTableInfo(name: "table_\($0)") } + await provider.resetForDatabase( + "shop", + tables: tables, + driver: MockDatabaseDriver(), + connection: TestFixtures.makeConnection() + ) + await provider.waitForEagerColumnLoad() + let fetchesBefore = fetches.count + + let builder = CopilotPreambleBuilder() + await builder.buildPreamble( + schemaProvider: provider, + databaseName: "shop", + databaseType: .mysql, + referencing: "SELECT * FROM table_1500 t JOIN table_1999 u ON u.id = t.id;", + maxTables: 20 + ) + + #expect(fetches.count - fetchesBefore <= 20) + let lines = builder.preamble.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) + try #require(lines.count == 25) + #expect(lines[3].hasPrefix("-- table_1500(")) + #expect(lines[4].hasPrefix("-- table_1999(")) + #expect(lines[5] == "-- line break(id INT PK NOT NULL)") + #expect(lines[23] == "-- … and 1980 more tables") + #expect(builder.preambleLineCount == 24) + } } diff --git a/apple/TableProTests/Core/LSP/LSPDocumentManagerTests.swift b/apple/TableProTests/Core/LSP/LSPDocumentManagerTests.swift new file mode 100644 index 0000000000..c9f6a62ef3 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPDocumentManagerTests.swift @@ -0,0 +1,41 @@ +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct LSPDocumentManagerTests { + @Test("a version starts at zero on open, rises by one per change and starts over on a reopen") + func versionsFollowOpenAndChange() { + let manager = LSPDocumentManager() + let uri = "file:///query-1.sql" + let edit = [LSPTextDocumentContentChangeEvent(text: "SELECT 1")] + + #expect(manager.changeDocument(uri: uri, changes: edit)?.textDocument.version == nil) + #expect(manager.openDocument(uri: uri, languageId: "sql", text: "").version == 0) + #expect(manager.changeDocument(uri: uri, changes: edit)?.textDocument.version == 1) + #expect(manager.changeDocument(uri: uri, changes: edit)?.textDocument.version == 2) + + _ = manager.openDocument(uri: uri, languageId: "sql", text: "") + #expect(manager.version(for: uri) == 0) + #expect(manager.closeDocument(uri: uri)?.uri == uri) + #expect(manager.version(for: uri) == nil) + #expect(manager.closeDocument(uri: uri)?.uri == nil) + } + + @Test("a full text change encodes no range, and a range change round-trips with it") + func changeEventEncodesItsRangeOnlyWhenSet() throws { + let full = try JSONSerialization.jsonObject( + with: JSONEncoder().encode(LSPTextDocumentContentChangeEvent(text: "x")) + ) as? [String: Any] + #expect(full?.keys.sorted() == ["text"]) + + let ranged = LSPTextDocumentContentChangeEvent( + range: LSPRange(start: LSPPosition(line: 3, character: 1), end: LSPPosition(line: 4, character: 0)), + text: "y" + ) + let data = try JSONEncoder().encode(ranged) + let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] + #expect(object?.keys.sorted() == ["range", "text"]) + #expect(try JSONDecoder().decode(LSPTextDocumentContentChangeEvent.self, from: data) == ranged) + } +} diff --git a/apple/TableProTests/Core/Vim/VimTextBufferAdapterLineIndexTests.swift b/apple/TableProTests/Core/Vim/VimTextBufferAdapterLineIndexTests.swift new file mode 100644 index 0000000000..5699a81d80 --- /dev/null +++ b/apple/TableProTests/Core/Vim/VimTextBufferAdapterLineIndexTests.swift @@ -0,0 +1,316 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTextEngine +import Testing + +private let lineIndexDocuments = [ + "", + "SELECT 1", + "a\nb\n", + "\n", + "\n\n", + "a\n\n\nb\n\n", + "line one\r\nline two\r\n\r\nline four", + "a\r\n", + "\r\n\r\n", + "a\r\nb\nc\r\n\nd", + "x\u{1F600}y\n\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467} z\ne\u{301}\n\u{1F600}", + "\tindent\n\t\tmore\n spaces\t\n\t", +] + +@MainActor +struct VimTextBufferAdapterLineIndexTests { + private final class StubDelegate: TextViewDelegate {} + + private func makeTextView(string: String) -> TextView { + let textView = TextView( + string: string, + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + textColor: .labelColor, + lineHeightMultiplier: 1.0, + wrapLines: false, + isEditable: true, + isSelectable: true, + letterSpacing: 1.0, + delegate: StubDelegate() + ) + textView.frame = NSRect(x: 0, y: 0, width: 500, height: 500) + textView.layout() + return textView + } + + // MARK: - Same answers as the line walk + + @Test("Line queries answer as the line walk does", arguments: lineIndexDocuments) + func freshDocumentMatchesTheLineWalk(text: String) { + let textView = makeTextView(string: text) + let adapter = VimTextBufferAdapter(textView: textView) + + let mismatches = withExtendedLifetime(textView) { Self.mismatches(adapter, against: textView.string as NSString) } + + #expect(mismatches.isEmpty, "\(mismatches.prefix(8))") + } + + @Test("Line queries answer as the line walk does after each of a run of edits") + func editsKeepLineQueriesInStep() { + let textView = makeTextView(string: "SELECT 1\nFROM t\r\n\nWHERE a = 1\n") + let adapter = VimTextBufferAdapter(textView: textView) + let replacements = ["", "", "a", "xy", "\n", "\r", "\r\n", "\n\n", "q\nr\r\ns", "\u{1F600}", "\t", "--\n"] + var generator = SeededGenerator(seed: 0x51DE) + var mismatches: [String] = [] + + withExtendedLifetime(textView) { + for step in 0..<60 { + let text = textView.string as NSString + // Edits land on character boundaries, as the editor's own do, so none leaves half a surrogate pair. + let start = Self.characterBoundary(Int.random(in: 0...text.length, using: &generator), in: text) + let end = max( + start, + Self.characterBoundary(min(text.length, start + Int.random(in: 0...6, using: &generator)), in: text) + ) + let replacement = replacements.randomElement(using: &generator) ?? "" + textView.replaceCharacters(in: NSRange(location: start, length: end - start), with: replacement) + // The editor drops the adapter's cache on every edit. + adapter.invalidateLineCache() + + let edited = textView.string as NSString + mismatches += Self.mismatches(adapter, against: edited).map { + "step \(step) \(String(reflecting: edited as String)): \($0)" + } + } + } + + #expect(mismatches.isEmpty, "\(mismatches.prefix(8))") + } + + @Test("Line queries walk the text while the line storage still describes the text before an edit", arguments: [ + ("a\nb\r\nc\n", NSRange(location: 2, length: 0), "x\n\ny\r\n"), + ("one\ntwo\nthree", NSRange(location: 0, length: 4), ""), + ("SELECT 1\r\n\r\nSELECT 2", NSRange(location: 10, length: 2), "--\n"), + ]) + func staleLineStorageFallsBackToTheText(original: String, range: NSRange, replacement: String) { + let textView = makeTextView(string: original) + let adapter = VimTextBufferAdapter(textView: textView) + + withExtendedLifetime(textView) { + textView.layoutManager.processesEdits = false + textView.textStorage.replaceCharacters(in: range, with: replacement) + #expect(textView.layoutManager.lineStorage.length != textView.textStorage.length) + + let stale = Self.mismatches(adapter, against: textView.string as NSString) + #expect(stale.isEmpty, "\(stale.prefix(8))") + + textView.layoutManager.processesEdits = true + textView.layoutManager.reset() + let rebuilt = Self.mismatches(adapter, against: textView.string as NSString) + #expect(rebuilt.isEmpty, "\(rebuilt.prefix(8))") + } + } + + // The line walk this replaced counted only line feeds for `lineAndColumn`, while `lineRange` and + // `offset(forLine:column:)` also break at a lone carriage return. Every query now numbers lines the same way. + @Test("A lone carriage return ends a line for every line query", arguments: [ + "a\rb\rc", + "a\rb\r", + "a\r\rb\nc\r\nd", + ]) + func carriageReturnEndsALine(text: String) { + let textView = makeTextView(string: text) + let adapter = VimTextBufferAdapter(textView: textView) + let nsText = text as NSString + var lineRanges: [NSRange] = [] + var location = 0 + while location < nsText.length { + let range = nsText.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0)) + lineRanges.append(range) + location = NSMaxRange(range) + } + + var mismatches: [String] = [] + withExtendedLifetime(textView) { + if adapter.lineCount != lineRanges.count { + mismatches.append("lineCount \(adapter.lineCount), expected \(lineRanges.count)") + } + for (line, range) in lineRanges.enumerated() { + if adapter.offset(forLine: line, column: 0) != range.location { + mismatches.append("offset(forLine: \(line)) \(adapter.offset(forLine: line, column: 0))") + } + for offset in range.location.. SectionScript { + let script = String(repeating: "SELECT id, name FROM users WHERE id = 1;\n", count: lines) + let textView = makeTextView(string: script) + let adapter = VimTextBufferAdapter(textView: textView) + return SectionScript( + textView: textView, + adapter: adapter, + engine: VimEngine(buffer: adapter), + lastOffset: (script as NSString).length - 1 + ) + } + + private static func pressSectionForward(_ engine: VimEngine, _ adapter: VimTextBufferAdapter) -> Int { + adapter.setSelectedRange(NSRange(location: 0, length: 0)) + _ = engine.process("]", shift: false) + _ = engine.process("]", shift: false) + return adapter.selectedRange().location + } + + private static func secondsSpent(_ work: () -> Void) -> TimeInterval { + let start = ProcessInfo.processInfo.systemUptime + work() + return ProcessInfo.processInfo.systemUptime - start + } + + // MARK: - Comparison + + private static func characterBoundary(_ offset: Int, in text: NSString) -> Int { + guard offset > 0, offset < text.length else { return offset } + return text.rangeOfComposedCharacterSequence(at: offset).location + } + + private static func mismatches(_ adapter: VimTextBufferAdapter, against text: NSString) -> [String] { + var found: [String] = [] + if adapter.length != text.length { + found.append("length \(adapter.length), expected \(text.length)") + } + let lineCount = LineWalk.lineCount(of: text) + if adapter.lineCount != lineCount { + found.append("lineCount \(adapter.lineCount), expected \(lineCount)") + } + for offset in -1...(text.length + 1) { + let position = adapter.lineAndColumn(forOffset: offset) + let expectedPosition = LineWalk.lineAndColumn(of: text, forOffset: offset) + if position != expectedPosition { + found.append("lineAndColumn(\(offset)) \(position), expected \(expectedPosition)") + } + let range = adapter.lineRange(forOffset: offset) + let expectedRange = LineWalk.lineRange(of: text, forOffset: offset) + if range != expectedRange { + found.append("lineRange(\(offset)) \(range), expected \(expectedRange)") + } + if offset >= 0 && offset < text.length && adapter.character(at: offset) != text.character(at: offset) { + found.append("character(at: \(offset)) \(adapter.character(at: offset))") + } + } + for line in -1...(lineCount + 1) { + for column in [-1, 0, 1, 2, 5, 1_000] { + let offset = adapter.offset(forLine: line, column: column) + let expectedOffset = LineWalk.offset(of: text, forLine: line, column: column) + if offset != expectedOffset { + found.append("offset(forLine: \(line), column: \(column)) \(offset), expected \(expectedOffset)") + } + } + } + return found + } + +} + +// The line walk `VimTextBufferAdapter` used before it read the layout manager's line storage, kept as the reference +// its answers are compared with. `lineAndColumn` counts every line break here, as the other queries always did; the +// old one counted only line feeds, which `carriageReturnEndsALine` covers. +private enum LineWalk { + static func lineCount(of text: NSString) -> Int { + if text.length == 0 { return 1 } + var count = 0 + var index = 0 + while index < text.length { + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) + count += 1 + index = lineRange.location + lineRange.length + } + return max(1, count) + } + + static func lineRange(of text: NSString, forOffset offset: Int) -> NSRange { + let clampedOffset = min(max(0, offset), text.length) + return text.lineRangeBreakingAtLineEndings(for: NSRange(location: clampedOffset, length: 0)) + } + + static func lineAndColumn(of text: NSString, forOffset offset: Int) -> (line: Int, column: Int) { + let clampedOffset = min(max(0, offset), text.length) + if text.length == 0 { return (0, 0) } + let safeOffset = min(clampedOffset, max(0, text.length - 1)) + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: safeOffset, length: 0)) + var line = 0 + var index = 0 + while index < lineRange.location { + index = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0))) + line += 1 + } + return (line, clampedOffset - lineRange.location) + } + + static func offset(of text: NSString, forLine line: Int, column: Int) -> Int { + var currentLine = 0 + var index = 0 + while index < text.length && currentLine < line { + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) + currentLine += 1 + index = lineRange.location + lineRange.length + } + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: min(index, text.length), length: 0)) + let contentLength: Int + let lineEnd = lineRange.location + lineRange.length + if lineEnd > lineRange.location && lineEnd <= text.length && text.character(at: lineEnd - 1) == 0x0A { + contentLength = lineRange.length - 1 + } else { + contentLength = lineRange.length + } + let clampedCol = min(column, max(0, contentLength - 1)) + return lineRange.location + max(0, clampedCol) + } +} diff --git a/apple/TableProTests/Models/QueryHistoryEntryTests.swift b/apple/TableProTests/Models/QueryHistoryEntryTests.swift index a1694e6a79..eaacf27606 100644 --- a/apple/TableProTests/Models/QueryHistoryEntryTests.swift +++ b/apple/TableProTests/Models/QueryHistoryEntryTests.swift @@ -1,16 +1,54 @@ -// -// QueryHistoryEntryTests.swift -// TableProTests -// - import Foundation -import TableProPluginKit @testable import TablePro +import TableProPluginKit import Testing +private let previewSamples: [String] = [ + "", + " ", + "\n", + " \t \r\n \u{2028} ", + "SELECT 1", + "SELECT 1;", + " SELECT 1; \n", + "\n\n SELECT 1\n\n\n", + "SELECT id,\r\n name\r\n FROM users\r\n", + "a\u{2028}b\u{2029}c\u{85}d\u{0B}e\u{0C}f\rg", + "\tSELECT\t1\t\n\t\tFROM t\t", + "\u{3000}SELECT\u{A0}1\u{2003}\n\u{A0}", + "abc\n\u{301}def", + "SELECT" + String(repeating: " ", count: 300) + "1", + "SELECT 1\u{37E}", + "SELECT 1\u{600}", + String(repeating: "a", count: 98) + "\u{600}", + String(repeating: "a", count: 99), + String(repeating: "a", count: 99) + ";", + String(repeating: "a", count: 100), + String(repeating: "a", count: 101), + String(repeating: "a", count: 98) + "\u{1F600}", + String(repeating: "a", count: 99) + "\u{1F600}", + String(repeating: "a", count: 100) + "\u{301}", + String(repeating: "a", count: 120) + "\u{37E}", + String(repeating: "a", count: 159), + String(repeating: "a", count: 160), + String(repeating: "a", count: 161), + String(repeating: "a", count: 158) + "\u{1F600}", + String(repeating: "a", count: 159) + "\u{1F600}", + String(repeating: "a", count: 159) + "e\u{301}x", + String(repeating: "a", count: 160) + "\u{301}", + String(repeating: "a", count: 158) + "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}b", + String(repeating: "a", count: 159) + "\u{1F1FA}\u{1F1F8}\u{1F1EC}\u{1F1E7}", + String(repeating: "a", count: 80) + "\n" + String(repeating: "b", count: 79) + "\n\u{301}c", + "a" + String(repeating: "\u{301}", count: 400) + " b\nc", + String(repeating: "a", count: 150) + String(repeating: " ", count: 200), + String(repeating: " SELECT col FROM t\r\n", count: 30), + String(repeating: "word ", count: 40) + "\n\n" + String(repeating: "\t", count: 50), +] + struct QueryHistoryEntryTests { private func makeEntry( query: String, + statementType: QueryHistoryStatementType? = nil, executionTime: TimeInterval = 0.1, rowCount: Int = 10, wasSuccessful: Bool = true @@ -21,6 +59,7 @@ struct QueryHistoryEntryTests { databaseName: "test", databaseType: .postgresql, source: .editor, + statementType: statementType, executionTime: executionTime, rowCount: rowCount, wasSuccessful: wasSuccessful @@ -45,6 +84,69 @@ struct QueryHistoryEntryTests { #expect(entry.singleLinePreview.contains("\n") == false) } + @Test("The previews read as splitting and rejoining the whole query did", arguments: previewSamples) + func previewsMatchTheWholeQueryVersion(query: String) { + let entry = makeEntry(query: query, statementType: .select) + #expect(Array(entry.singleLinePreview.unicodeScalars) == Array(WholeQueryPreview.singleLine(query).unicodeScalars)) + #expect(Array(entry.queryPreview.unicodeScalars) == Array(WholeQueryPreview.query(query).unicodeScalars)) + } + + @Test("The previews read as splitting and rejoining the whole query did on generated queries") + func previewsMatchTheWholeQueryVersionOnGeneratedQueries() { + let pieces = [ + "a", "SELECT", " ", " ", "\t", "\n", "\r", "\r\n", "\u{2028}", "\u{85}", "\u{3000}", "\u{A0}", + "\u{1F600}", "\u{1F468}\u{200D}\u{1F469}", "e\u{301}", "\u{301}", "\u{600}", ";", "\u{37E}", + "\u{1F1FA}\u{1F1F8}", "\u{0B}", + ] + var generator = SeededGenerator(seed: 0x5EED) + var mismatches: [String] = [] + for _ in 0..<500 { + var query = "" + for _ in 0.. 1_000_000) + + var shortPreviews = TimeInterval.infinity + var longPreviews = TimeInterval.infinity + for _ in 0..<5 { + shortPreviews = min(shortPreviews, Self.secondsSpent(previewing: short)) + longPreviews = min(longPreviews, Self.secondsSpent(previewing: long)) + } + + #expect(long.singleLinePreview == short.singleLinePreview) + #expect(long.queryPreview == short.queryPreview) + #expect(longPreviews < shortPreviews * 4) + } + + private static func secondsSpent(previewing entry: QueryHistoryEntry) -> TimeInterval { + let start = ProcessInfo.processInfo.systemUptime + for _ in 0..<100 { + _ = entry.singleLinePreview + _ = entry.queryPreview + } + return ProcessInfo.processInfo.systemUptime - start + } + @Test("statementType is classified from the query when not supplied") func statementTypeIsClassified() { #expect(makeEntry(query: "SELECT 1").statementType == .select) @@ -100,3 +202,30 @@ struct QueryHistoryEntryTests { #expect(entry.databaseDisplayName == "test") } } + +// The previews as they were before they stopped reading at the cut, kept as the reference the new ones are compared +// with. +private enum WholeQueryPreview { + static func singleLine(_ query: String) -> String { + let collapsed = query + .components(separatedBy: .newlines) + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + .joined(separator: " ") + if (collapsed as NSString).length > 160 { + return String(collapsed.prefix(160)) + "…" + } + return collapsed + } + + static func query(_ query: String) -> String { + var trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.hasSuffix(";") { + trimmed += ";" + } + if (trimmed as NSString).length > 100 { + return String(trimmed.prefix(100)) + "…" + } + return trimmed + } +} diff --git a/apple/TableProTests/Views/Editor/SQLFoldScannerPathTests.swift b/apple/TableProTests/Views/Editor/SQLFoldScannerPathTests.swift new file mode 100644 index 0000000000..1534746570 --- /dev/null +++ b/apple/TableProTests/Views/Editor/SQLFoldScannerPathTests.swift @@ -0,0 +1,193 @@ +import AppKit +import Combine +import Darwin +import Foundation +@testable import TablePro +@testable import TableProEditorKit +import TableProGrammars +import TableProSQLGrammar +import TableProTestSupport +import TableProTextEngine +import TableProUtilities +import Testing + +// SQL folding runs `SQLFoldScanner` off the main thread on a copy of the text. These hold that path to the folds the +// per-line walk gives, and the main thread to a copy per keystroke instead of a scan. +// Serialized: the keystroke case reads the main thread's CPU time, which a case running beside it would add to. +@MainActor +@Suite(.serialized) +struct SQLFoldScannerPathTests { + @MainActor + private final class LineWalkOnly: LineFoldProvider { + private let base: SQLLineFoldProvider + + init(grammar: SQLLexicalGrammar) { + base = SQLLineFoldProvider(grammar: grammar) + } + + func foldLevelAtLine( + lineNumber: Int, + lineRange: NSRange, + previousDepth: Int, + controller: TextViewController + ) -> [LineFoldProviderLineInfo] { + base.foldLevelAtLine( + lineNumber: lineNumber, + lineRange: lineRange, + previousDepth: previousDepth, + controller: controller + ) + } + } + + private struct Fold: Equatable { + let depth: Int + let range: Range + } + + private static let mixedScript = """ + CREATE TABLE users ( + id INT PRIMARY KEY, + name TEXT, + CHECK ( + id > 0 + ) + ); + /* a comment + over lines */ + SELECT id, + CASE + WHEN id > 1 THEN 'a' + ELSE 'b' + END AS label + FROM users + WHERE id IN ( + SELECT id FROM users + ); + DO $$ + BEGIN + PERFORM 1; + END + $$; + SELECT 1; SELECT (2); + INSERT INTO users VALUES + (1, 'a'), + (2, 'b'); + """ + + private func makeController(text: String, provider: LineFoldProvider) -> TextViewController { + let controller = TextViewController( + string: text, + language: .sql, + configuration: SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: .init(foldingSizeLimit: StatementRunController.defaultSizeLimit) + ), + cursorPositions: [], + highlightProviders: [], + foldProvider: provider + ) + controller.loadView() + controller.view.frame = NSRect(x: 0, y: 0, width: 1_000, height: 1_000) + controller.view.layoutSubtreeIfNeeded() + return controller + } + + private func folds(in controller: TextViewController) -> [Fold] { + controller.foldModel? + .getFolds(in: 0.. Double { + var info = thread_basic_info() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let thread = pthread_mach_thread_np(pthread_self()) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + thread_info(thread, thread_flavor_t(THREAD_BASIC_INFO), $0, &count) + } + } + guard result == KERN_SUCCESS else { return 0 } + return Double(info.user_time.seconds + info.system_time.seconds) * 1_000 + + Double(info.user_time.microseconds + info.system_time.microseconds) / 1_000 + } + + @Test("The scanner path folds a script the way the per-line walk does", arguments: ["\n", "\r\n"]) + func scannerMatchesLineWalk(lineEnding: String) async throws { + let text = Self.mixedScript.replacingOccurrences(of: "\n", with: lineEnding) + let scanned = makeController(text: text, provider: SQLLineFoldProvider(grammar: TestGrammar.postgres)) + let walked = makeController(text: text, provider: LineWalkOnly(grammar: TestGrammar.postgres)) + + try await waitUntil(timeout: .seconds(30)) { !folds(in: scanned).isEmpty && !folds(in: walked).isEmpty } + + #expect(folds(in: walked).count >= 6) + #expect(folds(in: scanned) == folds(in: walked)) + } + + // A pass used to walk every line through the provider on the main thread, the scan included; now the main thread + // only copies the text and takes the result. Both are timed on the same document in the same run, so a slow runner + // moves both sides. Long single-line statements keep the fold count, and the Debug fold store's fill time, low. + @Test("A fold pass costs the main thread a fraction of the per-line walk") + func passCostsAFractionOfTheWalk() async throws { + let statement = "SELECT '" + String(repeating: "x", count: 180) + "' FROM t;\n" + let block = "CREATE TABLE t (\n id INT\n);\n" + String(repeating: statement, count: 20) + let text = String(repeating: block, count: 300_000 / (block as NSString).length) + #expect((text as NSString).length > LineFoldModel.immediateRecalculationLimit) + + let walked = TextViewController(string: text, language: .sql, configuration: configuration, cursorPositions: []) + let walker = SQLLineFoldProvider(grammar: TestGrammar.postgres) + var walkCosts: [Double] = [] + for _ in 0..<3 { + var lines = walked.textView.layoutManager.lineStorage.makeIterator() + let start = Self.mainThreadCPUMilliseconds() + while let line = lines.next() { + _ = walker.foldLevelAtLine( + lineNumber: line.index, lineRange: line.range, previousDepth: 0, controller: walked + ) + } + walkCosts.append(Self.mainThreadCPUMilliseconds() - start) + } + + let scanned = TextViewController( + string: text, + language: .sql, + configuration: configuration, + cursorPositions: [], + foldProvider: SQLLineFoldProvider(grammar: TestGrammar.postgres) + ) + let model = LineFoldModel(controller: scanned, foldView: NSView()) + defer { model.destroy() } + try await waitUntil(timeout: .seconds(30)) { !model.getFolds(in: 0..<(text as NSString).length).isEmpty } + // Awaited, not polled: polling while the pass runs would bill the main thread for the wait. + let (published, publish) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + let subscription = model.$foldCache.dropFirst().sink { _ in publish.yield() } + defer { subscription.cancel() } + var passCosts: [Double] = [] + for _ in 0..<3 { + let start = Self.mainThreadCPUMilliseconds() + model.refresh() + for await _ in published { break } + passCosts.append(Self.mainThreadCPUMilliseconds() - start) + } + + let walk = walkCosts.sorted()[1] + let pass = passCosts.sorted()[1] + #expect(pass < walk / 4, "a pass took \(pass) ms of main thread, the walk \(walk) ms") + } + + private var configuration: SourceEditorConfiguration { + SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: .init(foldingSizeLimit: StatementRunController.defaultSizeLimit) + ) + } +} diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorCaretPublishTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorCaretPublishTests.swift new file mode 100644 index 0000000000..8be11ca00a --- /dev/null +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorCaretPublishTests.swift @@ -0,0 +1,27 @@ +import Combine +import Foundation +import SwiftUI +@testable import TablePro +import TableProEditorKit +import Testing + +// The query editor binds its caret to `$coordinator.cursorPositions`. Every view in the main window observes the +// coordinator, so a publish per caret move re-rendered the window on each arrow key. +@MainActor +struct MainContentCoordinatorCaretPublishTests { + @Test("Moving the caret writes through the editor's binding without publishing the coordinator") + func caretMoveDoesNotPublish() { + let fixture = CoordinatorFixture() + var publishes = 0 + let subscription = fixture.coordinator.objectWillChange.sink { _ in publishes += 1 } + defer { subscription.cancel() } + + let binding = ObservedObject(wrappedValue: fixture.coordinator).projectedValue.cursorPositions + for offset in 0..<20 { + binding.wrappedValue = [CursorPosition(range: NSRange(location: offset, length: 0))] + } + + #expect(publishes == 0) + #expect(fixture.coordinator.cursorPositions.map { $0.range } == [NSRange(location: 19, length: 0)]) + } +} diff --git a/apple/changelog.d/4218.changed.md b/apple/changelog.d/4218.changed.md new file mode 100644 index 0000000000..ecf1106a66 --- /dev/null +++ b/apple/changelog.d/4218.changed.md @@ -0,0 +1 @@ +Copilot's schema context covers up to Max schema tables tables, the ones the current statement names first. diff --git a/apple/changelog.d/4218.fixed.1.md b/apple/changelog.d/4218.fixed.1.md new file mode 100644 index 0000000000..32c2357d0b --- /dev/null +++ b/apple/changelog.d/4218.fixed.1.md @@ -0,0 +1 @@ +Vim section and paragraph motions froze on long scripts, and j and k slowed down as a script grew. diff --git a/apple/changelog.d/4218.fixed.2.md b/apple/changelog.d/4218.fixed.2.md new file mode 100644 index 0000000000..3b3fce3ea6 --- /dev/null +++ b/apple/changelog.d/4218.fixed.2.md @@ -0,0 +1 @@ +Copilot resent the whole script and schema on every keystroke, and stopped suggesting after a tab switch. diff --git a/apple/changelog.d/4218.fixed.3.md b/apple/changelog.d/4218.fixed.3.md new file mode 100644 index 0000000000..e653509202 --- /dev/null +++ b/apple/changelog.d/4218.fixed.3.md @@ -0,0 +1 @@ +Copilot ran one metadata query per table when an editor opened, thousands on a large database. diff --git a/apple/changelog.d/4218.fixed.4.md b/apple/changelog.d/4218.fixed.4.md new file mode 100644 index 0000000000..d71d686cde --- /dev/null +++ b/apple/changelog.d/4218.fixed.4.md @@ -0,0 +1 @@ +Moving the caret in the query editor redrew the whole window. diff --git a/apple/changelog.d/4218.fixed.5.md b/apple/changelog.d/4218.fixed.5.md new file mode 100644 index 0000000000..7a1b936cf4 --- /dev/null +++ b/apple/changelog.d/4218.fixed.5.md @@ -0,0 +1 @@ +The query history panel lagged on very long saved queries. diff --git a/apple/changelog.d/4218.fixed.md b/apple/changelog.d/4218.fixed.md new file mode 100644 index 0000000000..65870fa9d7 --- /dev/null +++ b/apple/changelog.d/4218.fixed.md @@ -0,0 +1 @@ +Typing lagged in large SQL scripts while code folding was on.